diff options
Diffstat (limited to 'drivers')
834 files changed, 49028 insertions, 12056 deletions
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 6078064684c6..ed3d2d1a7ae9 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) "ACPI: IORT: " fmt #include <linux/acpi_iort.h> +#include <linux/bitfield.h> #include <linux/iommu.h> #include <linux/kernel.h> #include <linux/list.h> @@ -902,9 +903,9 @@ static inline bool iort_iommu_driver_enabled(u8 type) { switch (type) { case ACPI_IORT_NODE_SMMU_V3: - return IS_BUILTIN(CONFIG_ARM_SMMU_V3); + return IS_ENABLED(CONFIG_ARM_SMMU_V3); case ACPI_IORT_NODE_SMMU: - return IS_BUILTIN(CONFIG_ARM_SMMU); + return IS_ENABLED(CONFIG_ARM_SMMU); default: pr_warn("IORT node type %u does not describe an SMMU\n", type); return false; @@ -976,6 +977,20 @@ static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data) return iort_iommu_xlate(info->dev, parent, streamid); } +static void iort_named_component_init(struct device *dev, + struct acpi_iort_node *node) +{ + struct acpi_iort_named_component *nc; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + + if (!fwspec) + return; + + nc = (struct acpi_iort_named_component *)node->node_data; + fwspec->num_pasid_bits = FIELD_GET(ACPI_IORT_NC_PASID_BITS, + nc->node_flags); +} + /** * iort_iommu_configure - Set-up IOMMU configuration for a device. * @@ -1030,6 +1045,9 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) if (parent) err = iort_iommu_xlate(dev, parent, streamid); } while (parent && !err); + + if (!err) + iort_named_component_init(dev, node); } /* diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 15cc7d5a6185..111a407dcc77 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -1202,13 +1202,12 @@ static int acpi_battery_alarm_proc_open(struct inode *inode, struct file *file) return single_open(file, acpi_battery_alarm_proc_show, PDE_DATA(inode)); } -static const struct file_operations acpi_battery_alarm_fops = { - .owner = THIS_MODULE, - .open = acpi_battery_alarm_proc_open, - .read = seq_read, - .write = acpi_battery_write_alarm, - .llseek = seq_lseek, - .release = single_release, +static const struct proc_ops acpi_battery_alarm_proc_ops = { + .proc_open = acpi_battery_alarm_proc_open, + .proc_read = seq_read, + .proc_write = acpi_battery_write_alarm, + .proc_lseek = seq_lseek, + .proc_release = single_release, }; static int acpi_battery_add_fs(struct acpi_device *device) @@ -1228,7 +1227,7 @@ static int acpi_battery_add_fs(struct acpi_device *device) acpi_battery_state_proc_show, acpi_driver_data(device))) return -ENODEV; if (!proc_create_data("alarm", S_IFREG | S_IRUGO | S_IWUSR, - acpi_device_dir(device), &acpi_battery_alarm_fops, + acpi_device_dir(device), &acpi_battery_alarm_proc_ops, acpi_driver_data(device))) return -ENODEV; return 0; diff --git a/drivers/acpi/proc.c b/drivers/acpi/proc.c index 652f19e6c541..0e62ef265ce4 100644 --- a/drivers/acpi/proc.c +++ b/drivers/acpi/proc.c @@ -136,18 +136,17 @@ acpi_system_wakeup_device_open_fs(struct inode *inode, struct file *file) PDE_DATA(inode)); } -static const struct file_operations acpi_system_wakeup_device_fops = { - .owner = THIS_MODULE, - .open = acpi_system_wakeup_device_open_fs, - .read = seq_read, - .write = acpi_system_write_wakeup_device, - .llseek = seq_lseek, - .release = single_release, +static const struct proc_ops acpi_system_wakeup_device_proc_ops = { + .proc_open = acpi_system_wakeup_device_open_fs, + .proc_read = seq_read, + .proc_write = acpi_system_write_wakeup_device, + .proc_lseek = seq_lseek, + .proc_release = single_release, }; void __init acpi_sleep_proc_init(void) { /* 'wakeup device' [R/W] */ proc_create("wakeup", S_IFREG | S_IRUGO | S_IWUSR, - acpi_root_dir, &acpi_system_wakeup_device_fops); + acpi_root_dir, &acpi_system_wakeup_device_proc_ops); } diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 915650bf519f..6d3448895382 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -1462,7 +1462,7 @@ int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr) iort_dma_setup(dev, &dma_addr, &size); iommu = iort_iommu_configure(dev); - if (IS_ERR(iommu) && PTR_ERR(iommu) == -EPROBE_DEFER) + if (PTR_ERR(iommu) == -EPROBE_DEFER) return -EPROBE_DEFER; arch_setup_dma_ops(dev, dma_addr, size, diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 4bfd1b14b390..11ea1aff40db 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -81,6 +81,7 @@ enum board_ids { static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent); static void ahci_remove_one(struct pci_dev *dev); +static void ahci_shutdown_one(struct pci_dev *dev); static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class, unsigned long deadline); static int ahci_avn_hardreset(struct ata_link *link, unsigned int *class, @@ -606,6 +607,7 @@ static struct pci_driver ahci_pci_driver = { .id_table = ahci_pci_tbl, .probe = ahci_init_one, .remove = ahci_remove_one, + .shutdown = ahci_shutdown_one, .driver = { .pm = &ahci_pci_pm_ops, }, @@ -1877,6 +1879,11 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; } +static void ahci_shutdown_one(struct pci_dev *pdev) +{ + ata_pci_shutdown_one(pdev); +} + static void ahci_remove_one(struct pci_dev *pdev) { pm_runtime_get_noresume(&pdev->dev); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 6f4ab5c5b52d..42c8728f6117 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -6767,6 +6767,26 @@ void ata_pci_remove_one(struct pci_dev *pdev) ata_host_detach(host); } +void ata_pci_shutdown_one(struct pci_dev *pdev) +{ + struct ata_host *host = pci_get_drvdata(pdev); + int i; + + for (i = 0; i < host->n_ports; i++) { + struct ata_port *ap = host->ports[i]; + + ap->pflags |= ATA_PFLAG_FROZEN; + + /* Disable port interrupts */ + if (ap->ops->freeze) + ap->ops->freeze(ap); + + /* Stop the port DMA engines */ + if (ap->ops->port_stop) + ap->ops->port_stop(ap); + } +} + /* move to PCI subsystem */ int pci_test_config_bits(struct pci_dev *pdev, const struct pci_bits *bits) { @@ -7387,6 +7407,7 @@ EXPORT_SYMBOL_GPL(ata_timing_cycle2mode); #ifdef CONFIG_PCI EXPORT_SYMBOL_GPL(pci_test_config_bits); +EXPORT_SYMBOL_GPL(ata_pci_shutdown_one); EXPORT_SYMBOL_GPL(ata_pci_remove_one); #ifdef CONFIG_PM EXPORT_SYMBOL_GPL(ata_pci_device_do_suspend); diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c index 391dff0f25a2..e9cf31f38450 100644 --- a/drivers/ata/pata_arasan_cf.c +++ b/drivers/ata/pata_arasan_cf.c @@ -526,9 +526,10 @@ static void data_xfer(struct work_struct *work) /* request dma channels */ /* dma_request_channel may sleep, so calling from process context */ - acdev->dma_chan = dma_request_slave_channel(acdev->host->dev, "data"); - if (!acdev->dma_chan) { + acdev->dma_chan = dma_request_chan(acdev->host->dev, "data"); + if (IS_ERR(acdev->dma_chan)) { dev_err(acdev->host->dev, "Unable to get dma_chan\n"); + acdev->dma_chan = NULL; goto chan_request_fail; } @@ -539,6 +540,7 @@ static void data_xfer(struct work_struct *work) } dma_release_channel(acdev->dma_chan); + acdev->dma_chan = NULL; /* data xferred successfully */ if (!ret) { diff --git a/drivers/ata/pata_pcmcia.c b/drivers/ata/pata_pcmcia.c index 3fe0754c0d52..8eb066abbd9c 100644 --- a/drivers/ata/pata_pcmcia.c +++ b/drivers/ata/pata_pcmcia.c @@ -309,6 +309,7 @@ static const struct pcmcia_device_id pcmcia_devices[] = { PCMCIA_DEVICE_MANF_CARD(0x0098, 0x0000), /* Toshiba */ PCMCIA_DEVICE_MANF_CARD(0x00a4, 0x002d), PCMCIA_DEVICE_MANF_CARD(0x00ce, 0x0000), /* Samsung */ + PCMCIA_DEVICE_MANF_CARD(0x00f1, 0x0101), /* SanDisk High (>8G) CFA */ PCMCIA_DEVICE_MANF_CARD(0x0319, 0x0000), /* Hitachi */ PCMCIA_DEVICE_MANF_CARD(0x2080, 0x0001), PCMCIA_DEVICE_MANF_CARD(0x4e01, 0x0100), /* Viking CFA */ diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 15659306ad69..b9f474c11393 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -376,7 +376,6 @@ static ssize_t valid_zones_show(struct device *dev, struct memory_block *mem = to_memory_block(dev); unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; - unsigned long valid_start_pfn, valid_end_pfn; struct zone *default_zone; int nid; @@ -389,11 +388,11 @@ static ssize_t valid_zones_show(struct device *dev, * The block contains more than one zone can not be offlined. * This can happen e.g. for ZONE_DMA and ZONE_DMA32 */ - if (!test_pages_in_a_zone(start_pfn, start_pfn + nr_pages, - &valid_start_pfn, &valid_end_pfn)) + default_zone = test_pages_in_a_zone(start_pfn, + start_pfn + nr_pages); + if (!default_zone) return sprintf(buf, "none\n"); - start_pfn = valid_start_pfn; - strcat(buf, page_zone(pfn_to_page(start_pfn))->name); + strcat(buf, default_zone->name); goto out; } diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 1bb8ec575352..025b1b77b11a 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -432,16 +432,6 @@ config VIRTIO_BLK This is the virtual block driver for virtio. It can be used with QEMU based VMMs (like KVM or Xen). Say Y or M. -config VIRTIO_BLK_SCSI - bool "SCSI passthrough request for the Virtio block driver" - depends on VIRTIO_BLK - select BLK_SCSI_REQUEST - ---help--- - Enable support for SCSI passthrough (e.g. the SG_IO ioctl) on - virtio-blk devices. This is only supported for the legacy - virtio protocol and not enabled by default by any hypervisor. - You probably want to use virtio-scsi instead. - config BLK_DEV_RBD tristate "Rados block device (RBD)" depends on INET && BLOCK diff --git a/drivers/block/brd.c b/drivers/block/brd.c index a8730cc4db10..220c5e18aba0 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -473,6 +473,25 @@ static struct kobject *brd_probe(dev_t dev, int *part, void *data) return kobj; } +static inline void brd_check_and_reset_par(void) +{ + if (unlikely(!max_part)) + max_part = 1; + + /* + * make sure 'max_part' can be divided exactly by (1U << MINORBITS), + * otherwise, it is possiable to get same dev_t when adding partitions. + */ + if ((1U << MINORBITS) % max_part != 0) + max_part = 1UL << fls(max_part); + + if (max_part > DISK_MAX_PARTS) { + pr_info("brd: max_part can't be larger than %d, reset max_part = %d.\n", + DISK_MAX_PARTS, DISK_MAX_PARTS); + max_part = DISK_MAX_PARTS; + } +} + static int __init brd_init(void) { struct brd_device *brd, *next; @@ -496,8 +515,7 @@ static int __init brd_init(void) if (register_blkdev(RAMDISK_MAJOR, "ramdisk")) return -EIO; - if (unlikely(!max_part)) - max_part = 1; + brd_check_and_reset_par(); for (i = 0; i < rd_nr; i++) { brd = brd_alloc(i); diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index ddbf56014c51..aae99a2d7bd4 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -622,7 +622,7 @@ struct fifo_buffer { int total; /* sum of all values */ int values[0]; }; -extern struct fifo_buffer *fifo_alloc(int fifo_size); +extern struct fifo_buffer *fifo_alloc(unsigned int fifo_size); /* flag bits per connection */ enum { diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index de2f94d0103a..da4a3ebe04ef 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -1575,7 +1575,8 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) struct drbd_device *device; struct disk_conf *new_disk_conf, *old_disk_conf; struct fifo_buffer *old_plan = NULL, *new_plan = NULL; - int err, fifo_size; + int err; + unsigned int fifo_size; retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); if (!adm_ctx.reply_skb) diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 2b3103c30857..79e216446030 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -3887,7 +3887,7 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i struct disk_conf *old_disk_conf = NULL, *new_disk_conf = NULL; const int apv = connection->agreed_pro_version; struct fifo_buffer *old_plan = NULL, *new_plan = NULL; - int fifo_size = 0; + unsigned int fifo_size = 0; int err; peer_device = conn_peer_device(connection, pi->vnr); diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index 5bdcc70ad589..b7f605c6e231 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c @@ -482,11 +482,11 @@ static void fifo_add_val(struct fifo_buffer *fb, int value) fb->values[i] += value; } -struct fifo_buffer *fifo_alloc(int fifo_size) +struct fifo_buffer *fifo_alloc(unsigned int fifo_size) { struct fifo_buffer *fb; - fb = kzalloc(sizeof(struct fifo_buffer) + sizeof(int) * fifo_size, GFP_NOIO); + fb = kzalloc(struct_size(fb, values, fifo_size), GFP_NOIO); if (!fb) return NULL; diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index b4607dd96185..78181908f0df 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1265,6 +1265,16 @@ static int nbd_start_device(struct nbd_device *nbd) args = kzalloc(sizeof(*args), GFP_KERNEL); if (!args) { sock_shutdown(nbd); + /* + * If num_connections is m (2 < m), + * and NO.1 ~ NO.n(1 < n < m) kzallocs are successful. + * But NO.(n + 1) failed. We still have n recv threads. + * So, add flush_workqueue here to prevent recv threads + * dropping the last config_refs and trying to destroy + * the workqueue from inside the workqueue. + */ + if (i) + flush_workqueue(nbd->recv_workq); return -ENOMEM; } sk_set_memalloc(config->socks[i]->sock->sk); diff --git a/drivers/block/null_blk_main.c b/drivers/block/null_blk_main.c index ae8d4bc532b0..16510795e377 100644 --- a/drivers/block/null_blk_main.c +++ b/drivers/block/null_blk_main.c @@ -263,34 +263,34 @@ static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, } /* The following macro should only be used with TYPE = {uint, ulong, bool}. */ -#define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ -static ssize_t \ -nullb_device_##NAME##_show(struct config_item *item, char *page) \ -{ \ - return nullb_device_##TYPE##_attr_show( \ - to_nullb_device(item)->NAME, page); \ -} \ -static ssize_t \ -nullb_device_##NAME##_store(struct config_item *item, const char *page, \ - size_t count) \ -{ \ - int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY; \ - struct nullb_device *dev = to_nullb_device(item); \ - TYPE new_value; \ - int ret; \ - \ - ret = nullb_device_##TYPE##_attr_store(&new_value, page, count); \ - if (ret < 0) \ - return ret; \ - if (apply_fn) \ - ret = apply_fn(dev, new_value); \ - else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ - ret = -EBUSY; \ - if (ret < 0) \ - return ret; \ - dev->NAME = new_value; \ - return count; \ -} \ +#define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ +static ssize_t \ +nullb_device_##NAME##_show(struct config_item *item, char *page) \ +{ \ + return nullb_device_##TYPE##_attr_show( \ + to_nullb_device(item)->NAME, page); \ +} \ +static ssize_t \ +nullb_device_##NAME##_store(struct config_item *item, const char *page, \ + size_t count) \ +{ \ + int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\ + struct nullb_device *dev = to_nullb_device(item); \ + TYPE uninitialized_var(new_value); \ + int ret; \ + \ + ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\ + if (ret < 0) \ + return ret; \ + if (apply_fn) \ + ret = apply_fn(dev, new_value); \ + else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ + ret = -EBUSY; \ + if (ret < 0) \ + return ret; \ + dev->NAME = new_value; \ + return count; \ +} \ CONFIGFS_ATTR(nullb_device_, NAME); static int nullb_apply_submit_queues(struct nullb_device *dev, diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 2b184563cd32..405b66e09040 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -2662,7 +2662,7 @@ static int rbd_img_fill_nodata(struct rbd_img_request *img_req, u64 off, u64 len) { struct ceph_file_extent ex = { off, len }; - union rbd_img_fill_iter dummy; + union rbd_img_fill_iter dummy = {}; struct rbd_img_fill_ctx fctx = { .pos_type = OBJ_REQUEST_NODATA, .pos = &dummy, @@ -7143,7 +7143,7 @@ static ssize_t do_rbd_add(struct bus_type *bus, if (rc) goto err_out_image_lock; - add_disk(rbd_dev->disk); + device_add_disk(&rbd_dev->dev, rbd_dev->disk, NULL); /* see rbd_init_disk() */ blk_put_queue(rbd_dev->disk->queue); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index fbbf18ac1d5d..54158766334b 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -11,7 +11,6 @@ #include <linux/virtio_blk.h> #include <linux/scatterlist.h> #include <linux/string_helpers.h> -#include <scsi/scsi_cmnd.h> #include <linux/idr.h> #include <linux/blk-mq.h> #include <linux/blk-mq-virtio.h> @@ -56,11 +55,6 @@ struct virtio_blk { }; struct virtblk_req { -#ifdef CONFIG_VIRTIO_BLK_SCSI - struct scsi_request sreq; /* for SCSI passthrough, must be first */ - u8 sense[SCSI_SENSE_BUFFERSIZE]; - struct virtio_scsi_inhdr in_hdr; -#endif struct virtio_blk_outhdr out_hdr; u8 status; struct scatterlist sg[]; @@ -78,80 +72,6 @@ static inline blk_status_t virtblk_result(struct virtblk_req *vbr) } } -/* - * If this is a packet command we need a couple of additional headers. Behind - * the normal outhdr we put a segment with the scsi command block, and before - * the normal inhdr we put the sense data and the inhdr with additional status - * information. - */ -#ifdef CONFIG_VIRTIO_BLK_SCSI -static int virtblk_add_req_scsi(struct virtqueue *vq, struct virtblk_req *vbr, - struct scatterlist *data_sg, bool have_data) -{ - struct scatterlist hdr, status, cmd, sense, inhdr, *sgs[6]; - unsigned int num_out = 0, num_in = 0; - - sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); - sgs[num_out++] = &hdr; - sg_init_one(&cmd, vbr->sreq.cmd, vbr->sreq.cmd_len); - sgs[num_out++] = &cmd; - - if (have_data) { - if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) - sgs[num_out++] = data_sg; - else - sgs[num_out + num_in++] = data_sg; - } - - sg_init_one(&sense, vbr->sense, SCSI_SENSE_BUFFERSIZE); - sgs[num_out + num_in++] = &sense; - sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr)); - sgs[num_out + num_in++] = &inhdr; - sg_init_one(&status, &vbr->status, sizeof(vbr->status)); - sgs[num_out + num_in++] = &status; - - return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); -} - -static inline void virtblk_scsi_request_done(struct request *req) -{ - struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); - struct virtio_blk *vblk = req->q->queuedata; - struct scsi_request *sreq = &vbr->sreq; - - sreq->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual); - sreq->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len); - sreq->result = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors); -} - -static int virtblk_ioctl(struct block_device *bdev, fmode_t mode, - unsigned int cmd, unsigned long data) -{ - struct gendisk *disk = bdev->bd_disk; - struct virtio_blk *vblk = disk->private_data; - - /* - * Only allow the generic SCSI ioctls if the host can support it. - */ - if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI)) - return -ENOTTY; - - return scsi_cmd_blk_ioctl(bdev, mode, cmd, - (void __user *)data); -} -#else -static inline int virtblk_add_req_scsi(struct virtqueue *vq, - struct virtblk_req *vbr, struct scatterlist *data_sg, - bool have_data) -{ - return -EIO; -} -static inline void virtblk_scsi_request_done(struct request *req) -{ -} -#define virtblk_ioctl NULL -#endif /* CONFIG_VIRTIO_BLK_SCSI */ - static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, struct scatterlist *data_sg, bool have_data) { @@ -216,13 +136,6 @@ static inline void virtblk_request_done(struct request *req) req->special_vec.bv_offset); } - switch (req_op(req)) { - case REQ_OP_SCSI_IN: - case REQ_OP_SCSI_OUT: - virtblk_scsi_request_done(req); - break; - } - blk_mq_end_request(req, virtblk_result(vbr)); } @@ -299,10 +212,6 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, type = VIRTIO_BLK_T_WRITE_ZEROES; unmap = !(req->cmd_flags & REQ_NOUNMAP); break; - case REQ_OP_SCSI_IN: - case REQ_OP_SCSI_OUT: - type = VIRTIO_BLK_T_SCSI_CMD; - break; case REQ_OP_DRV_IN: type = VIRTIO_BLK_T_GET_ID; break; @@ -333,10 +242,7 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, } spin_lock_irqsave(&vblk->vqs[qid].lock, flags); - if (blk_rq_is_scsi(req)) - err = virtblk_add_req_scsi(vblk->vqs[qid].vq, vbr, vbr->sg, num); - else - err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); + err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num); if (err) { virtqueue_kick(vblk->vqs[qid].vq); blk_mq_stop_hw_queue(hctx); @@ -404,10 +310,6 @@ static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) } static const struct block_device_operations virtblk_fops = { - .ioctl = virtblk_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = blkdev_compat_ptr_ioctl, -#endif .owner = THIS_MODULE, .getgeo = virtblk_getgeo, }; @@ -686,9 +588,6 @@ static int virtblk_init_request(struct blk_mq_tag_set *set, struct request *rq, struct virtio_blk *vblk = set->driver_data; struct virtblk_req *vbr = blk_mq_rq_to_pdu(rq); -#ifdef CONFIG_VIRTIO_BLK_SCSI - vbr->sreq.sense = vbr->sense; -#endif sg_init_table(vbr->sg, vblk->sg_elems); return 0; } @@ -701,23 +600,11 @@ static int virtblk_map_queues(struct blk_mq_tag_set *set) vblk->vdev, 0); } -#ifdef CONFIG_VIRTIO_BLK_SCSI -static void virtblk_initialize_rq(struct request *req) -{ - struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); - - scsi_req_init(&vbr->sreq); -} -#endif - static const struct blk_mq_ops virtio_mq_ops = { .queue_rq = virtio_queue_rq, .commit_rqs = virtio_commit_rqs, .complete = virtblk_request_done, .init_request = virtblk_init_request, -#ifdef CONFIG_VIRTIO_BLK_SCSI - .initialize_rq_fn = virtblk_initialize_rq, -#endif .map_queues = virtblk_map_queues, }; @@ -994,9 +881,6 @@ static const struct virtio_device_id id_table[] = { static unsigned int features_legacy[] = { VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, -#ifdef CONFIG_VIRTIO_BLK_SCSI - VIRTIO_BLK_F_SCSI, -#endif VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, } diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index 716b99aa2307..c2f71265af4b 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -62,8 +62,8 @@ * IO workloads. */ -static int xen_blkif_max_buffer_pages = 1024; -module_param_named(max_buffer_pages, xen_blkif_max_buffer_pages, int, 0644); +static int max_buffer_pages = 1024; +module_param_named(max_buffer_pages, max_buffer_pages, int, 0644); MODULE_PARM_DESC(max_buffer_pages, "Maximum number of free pages to keep in each block backend buffer"); @@ -78,8 +78,8 @@ MODULE_PARM_DESC(max_buffer_pages, * algorithm. */ -static int xen_blkif_max_pgrants = 1056; -module_param_named(max_persistent_grants, xen_blkif_max_pgrants, int, 0644); +static int max_pgrants = 1056; +module_param_named(max_persistent_grants, max_pgrants, int, 0644); MODULE_PARM_DESC(max_persistent_grants, "Maximum number of grants to map persistently"); @@ -88,8 +88,8 @@ MODULE_PARM_DESC(max_persistent_grants, * use. The time is in seconds, 0 means indefinitely long. */ -static unsigned int xen_blkif_pgrant_timeout = 60; -module_param_named(persistent_grant_unused_seconds, xen_blkif_pgrant_timeout, +static unsigned int pgrant_timeout = 60; +module_param_named(persistent_grant_unused_seconds, pgrant_timeout, uint, 0644); MODULE_PARM_DESC(persistent_grant_unused_seconds, "Time in seconds an unused persistent grant is allowed to " @@ -137,9 +137,8 @@ module_param(log_stats, int, 0644); static inline bool persistent_gnt_timeout(struct persistent_gnt *persistent_gnt) { - return xen_blkif_pgrant_timeout && - (jiffies - persistent_gnt->last_used >= - HZ * xen_blkif_pgrant_timeout); + return pgrant_timeout && (jiffies - persistent_gnt->last_used >= + HZ * pgrant_timeout); } static inline int get_free_page(struct xen_blkif_ring *ring, struct page **page) @@ -234,7 +233,7 @@ static int add_persistent_gnt(struct xen_blkif_ring *ring, struct persistent_gnt *this; struct xen_blkif *blkif = ring->blkif; - if (ring->persistent_gnt_c >= xen_blkif_max_pgrants) { + if (ring->persistent_gnt_c >= max_pgrants) { if (!blkif->vbd.overflow_max_grants) blkif->vbd.overflow_max_grants = 1; return -EBUSY; @@ -397,14 +396,13 @@ static void purge_persistent_gnt(struct xen_blkif_ring *ring) goto out; } - if (ring->persistent_gnt_c < xen_blkif_max_pgrants || - (ring->persistent_gnt_c == xen_blkif_max_pgrants && + if (ring->persistent_gnt_c < max_pgrants || + (ring->persistent_gnt_c == max_pgrants && !ring->blkif->vbd.overflow_max_grants)) { num_clean = 0; } else { - num_clean = (xen_blkif_max_pgrants / 100) * LRU_PERCENT_CLEAN; - num_clean = ring->persistent_gnt_c - xen_blkif_max_pgrants + - num_clean; + num_clean = (max_pgrants / 100) * LRU_PERCENT_CLEAN; + num_clean = ring->persistent_gnt_c - max_pgrants + num_clean; num_clean = min(ring->persistent_gnt_c, num_clean); pr_debug("Going to purge at least %u persistent grants\n", num_clean); @@ -599,8 +597,7 @@ static void print_stats(struct xen_blkif_ring *ring) current->comm, ring->st_oo_req, ring->st_rd_req, ring->st_wr_req, ring->st_f_req, ring->st_ds_req, - ring->persistent_gnt_c, - xen_blkif_max_pgrants); + ring->persistent_gnt_c, max_pgrants); ring->st_print = jiffies + msecs_to_jiffies(10 * 1000); ring->st_rd_req = 0; ring->st_wr_req = 0; @@ -656,8 +653,11 @@ purge_gnt_list: ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL); } - /* Shrink if we have more than xen_blkif_max_buffer_pages */ - shrink_free_pagepool(ring, xen_blkif_max_buffer_pages); + /* Shrink the free pages pool if it is too large. */ + if (time_before(jiffies, blkif->buffer_squeeze_end)) + shrink_free_pagepool(ring, 0); + else + shrink_free_pagepool(ring, max_buffer_pages); if (log_stats && time_after(jiffies, ring->st_print)) print_stats(ring); @@ -884,7 +884,7 @@ again: continue; } if (use_persistent_gnts && - ring->persistent_gnt_c < xen_blkif_max_pgrants) { + ring->persistent_gnt_c < max_pgrants) { /* * We are using persistent grants, the grant is * not mapped but we might have room for it. @@ -911,7 +911,7 @@ again: pages[seg_idx]->persistent_gnt = persistent_gnt; pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n", persistent_gnt->gnt, ring->persistent_gnt_c, - xen_blkif_max_pgrants); + max_pgrants); goto next; } if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) { diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h index 49132b0adbbe..a3eeccf3ac5f 100644 --- a/drivers/block/xen-blkback/common.h +++ b/drivers/block/xen-blkback/common.h @@ -319,6 +319,7 @@ struct xen_blkif { /* All rings for this device. */ struct xen_blkif_ring *rings; unsigned int nr_rings; + unsigned long buffer_squeeze_end; }; struct seg_buf { diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index 4c5d99f87813..42944d41aea0 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -467,7 +467,6 @@ static void xenvbd_sysfs_delif(struct xenbus_device *dev) device_remove_file(&dev->dev, &dev_attr_physical_device); } - static void xen_vbd_free(struct xen_vbd *vbd) { if (vbd->bdev) @@ -524,6 +523,7 @@ static int xen_vbd_create(struct xen_blkif *blkif, blkif_vdev_t handle, handle, blkif->domid); return 0; } + static int xen_blkbk_remove(struct xenbus_device *dev) { struct backend_info *be = dev_get_drvdata(&dev->dev); @@ -607,6 +607,7 @@ static void xen_blkbk_discard(struct xenbus_transaction xbt, struct backend_info if (err) dev_warn(&dev->dev, "writing feature-discard (%d)", err); } + int xen_blkbk_barrier(struct xenbus_transaction xbt, struct backend_info *be, int state) { @@ -691,7 +692,6 @@ fail: return err; } - /* * Callback received when the hotplug scripts have placed the physical-device * node. Read it and the mode node, and create a vbd. If the frontend is @@ -783,7 +783,6 @@ static void backend_changed(struct xenbus_watch *watch, } } - /* * Callback received when the frontend's state changes. */ @@ -858,9 +857,27 @@ static void frontend_changed(struct xenbus_device *dev, } } +/* Once a memory pressure is detected, squeeze free page pools for a while. */ +static unsigned int buffer_squeeze_duration_ms = 10; +module_param_named(buffer_squeeze_duration_ms, + buffer_squeeze_duration_ms, int, 0644); +MODULE_PARM_DESC(buffer_squeeze_duration_ms, +"Duration in ms to squeeze pages buffer when a memory pressure is detected"); -/* ** Connection ** */ +/* + * Callback received when the memory pressure is detected. + */ +static void reclaim_memory(struct xenbus_device *dev) +{ + struct backend_info *be = dev_get_drvdata(&dev->dev); + if (!be) + return; + be->blkif->buffer_squeeze_end = jiffies + + msecs_to_jiffies(buffer_squeeze_duration_ms); +} + +/* ** Connection ** */ /* * Write the physical details regarding the block device to the store, and @@ -1152,6 +1169,7 @@ static struct xenbus_driver xen_blkbk_driver = { .remove = xen_blkbk_remove, .otherend_changed = frontend_changed, .allow_rebind = true, + .reclaim_memory = reclaim_memory, }; int xen_blkif_xenbus_init(void) diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 57d50c5ba309..e2ad6bba2281 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -151,9 +151,6 @@ MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the #define BLK_RING_SIZE(info) \ __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages) -#define BLK_MAX_RING_SIZE \ - __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * XENBUS_MAX_RING_GRANTS) - /* * ring-ref%u i=(-1UL) would take 11 characters + 'ring-ref' is 8, so 19 * characters are enough. Define to 20 to keep consistent with backend. @@ -177,12 +174,12 @@ struct blkfront_ring_info { unsigned int evtchn, irq; struct work_struct work; struct gnttab_free_callback callback; - struct blk_shadow shadow[BLK_MAX_RING_SIZE]; struct list_head indirect_pages; struct list_head grants; unsigned int persistent_gnts_c; unsigned long shadow_free; struct blkfront_info *dev_info; + struct blk_shadow shadow[]; }; /* @@ -1915,7 +1912,8 @@ static int negotiate_mq(struct blkfront_info *info) info->nr_rings = 1; info->rinfo = kvcalloc(info->nr_rings, - sizeof(struct blkfront_ring_info), + struct_size(info->rinfo, shadow, + BLK_RING_SIZE(info)), GFP_KERNEL); if (!info->rinfo) { xenbus_dev_fatal(info->xbdev, -ENOMEM, "allocating ring_info structure"); diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c index aed2c45f7968..ed3b7dab678d 100644 --- a/drivers/char/hpet.c +++ b/drivers/char/hpet.c @@ -855,7 +855,7 @@ int hpet_alloc(struct hpet_data *hdp) return 0; } - hpetp = kzalloc(struct_size(hpetp, hp_dev, hdp->hd_nirqs - 1), + hpetp = kzalloc(struct_size(hpetp, hp_dev, hdp->hd_nirqs), GFP_KERNEL); if (!hpetp) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index d2a5791eb49f..cbf5eaea662c 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -157,7 +157,7 @@ static int bcm2835_rng_probe(struct platform_device *pdev) /* Clock is optional on most platforms */ priv->clk = devm_clk_get(dev, NULL); - if (IS_ERR(priv->clk) && PTR_ERR(priv->clk) == -EPROBE_DEFER) + if (PTR_ERR(priv->clk) == -EPROBE_DEFER) return -EPROBE_DEFER; priv->rng.name = pdev->name; diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index 0ed07d16ec8e..65952393e1bb 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -476,7 +476,7 @@ static int omap_rng_probe(struct platform_device *pdev) } priv->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(priv->clk) && PTR_ERR(priv->clk) == -EPROBE_DEFER) + if (PTR_ERR(priv->clk) == -EPROBE_DEFER) return -EPROBE_DEFER; if (!IS_ERR(priv->clk)) { ret = clk_prepare_enable(priv->clk); @@ -488,7 +488,7 @@ static int omap_rng_probe(struct platform_device *pdev) } priv->clk_reg = devm_clk_get(&pdev->dev, "reg"); - if (IS_ERR(priv->clk_reg) && PTR_ERR(priv->clk_reg) == -EPROBE_DEFER) + if (PTR_ERR(priv->clk_reg) == -EPROBE_DEFER) return -EPROBE_DEFER; if (!IS_ERR(priv->clk_reg)) { ret = clk_prepare_enable(priv->clk_reg); diff --git a/drivers/char/random.c b/drivers/char/random.c index cda12933a17d..c7f9584de2c8 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -307,6 +307,8 @@ * Eastlake, Steve Crocker, and Jeff Schiller. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/utsname.h> #include <linux/module.h> #include <linux/kernel.h> @@ -354,7 +356,6 @@ #define INPUT_POOL_WORDS (1 << (INPUT_POOL_SHIFT-5)) #define OUTPUT_POOL_SHIFT 10 #define OUTPUT_POOL_WORDS (1 << (OUTPUT_POOL_SHIFT-5)) -#define SEC_XFER_SIZE 512 #define EXTRACT_SIZE 10 @@ -371,12 +372,6 @@ #define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT) /* - * The minimum number of bits of entropy before we wake up a read on - * /dev/random. Should be enough to do a significant reseed. - */ -static int random_read_wakeup_bits = 64; - -/* * If the entropy count falls under this number of bits, then we * should wake up processes which are selecting or polling on write * access to /dev/random. @@ -436,42 +431,11 @@ static const struct poolinfo { /* was: x^128 + x^103 + x^76 + x^51 +x^25 + x + 1 */ /* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */ { S(128), 104, 76, 51, 25, 1 }, - /* was: x^32 + x^26 + x^20 + x^14 + x^7 + x + 1 */ - /* x^32 + x^26 + x^19 + x^14 + x^7 + x + 1 */ - { S(32), 26, 19, 14, 7, 1 }, -#if 0 - /* x^2048 + x^1638 + x^1231 + x^819 + x^411 + x + 1 -- 115 */ - { S(2048), 1638, 1231, 819, 411, 1 }, - - /* x^1024 + x^817 + x^615 + x^412 + x^204 + x + 1 -- 290 */ - { S(1024), 817, 615, 412, 204, 1 }, - - /* x^1024 + x^819 + x^616 + x^410 + x^207 + x^2 + 1 -- 115 */ - { S(1024), 819, 616, 410, 207, 2 }, - - /* x^512 + x^411 + x^308 + x^208 + x^104 + x + 1 -- 225 */ - { S(512), 411, 308, 208, 104, 1 }, - - /* x^512 + x^409 + x^307 + x^206 + x^102 + x^2 + 1 -- 95 */ - { S(512), 409, 307, 206, 102, 2 }, - /* x^512 + x^409 + x^309 + x^205 + x^103 + x^2 + 1 -- 95 */ - { S(512), 409, 309, 205, 103, 2 }, - - /* x^256 + x^205 + x^155 + x^101 + x^52 + x + 1 -- 125 */ - { S(256), 205, 155, 101, 52, 1 }, - - /* x^128 + x^103 + x^78 + x^51 + x^27 + x^2 + 1 -- 70 */ - { S(128), 103, 78, 51, 27, 2 }, - - /* x^64 + x^52 + x^39 + x^26 + x^14 + x + 1 -- 15 */ - { S(64), 52, 39, 26, 14, 1 }, -#endif }; /* * Static global variables */ -static DECLARE_WAIT_QUEUE_HEAD(random_read_wait); static DECLARE_WAIT_QUEUE_HEAD(random_write_wait); static struct fasync_struct *fasync; @@ -530,11 +494,8 @@ struct entropy_store { const struct poolinfo *poolinfo; __u32 *pool; const char *name; - struct entropy_store *pull; - struct work_struct push_work; /* read-write data: */ - unsigned long last_pulled; spinlock_t lock; unsigned short add_ptr; unsigned short input_rotate; @@ -550,9 +511,7 @@ static ssize_t _extract_entropy(struct entropy_store *r, void *buf, size_t nbytes, int fips); static void crng_reseed(struct crng_state *crng, struct entropy_store *r); -static void push_to_pool(struct work_struct *work); static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy; -static __u32 blocking_pool_data[OUTPUT_POOL_WORDS] __latent_entropy; static struct entropy_store input_pool = { .poolinfo = &poolinfo_table[0], @@ -561,16 +520,6 @@ static struct entropy_store input_pool = { .pool = input_pool_data }; -static struct entropy_store blocking_pool = { - .poolinfo = &poolinfo_table[1], - .name = "blocking", - .pull = &input_pool, - .lock = __SPIN_LOCK_UNLOCKED(blocking_pool.lock), - .pool = blocking_pool_data, - .push_work = __WORK_INITIALIZER(blocking_pool.push_work, - push_to_pool), -}; - static __u32 const twist_table[8] = { 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 }; @@ -759,22 +708,17 @@ retry: } while (unlikely(entropy_count < pool_size-2 && pnfrac)); } - if (unlikely(entropy_count < 0)) { - pr_warn("random: negative entropy/overflow: pool %s count %d\n", + if (WARN_ON(entropy_count < 0)) { + pr_warn("negative entropy/overflow: pool %s count %d\n", r->name, entropy_count); - WARN_ON(1); entropy_count = 0; } else if (entropy_count > pool_size) entropy_count = pool_size; - if ((r == &blocking_pool) && !r->initialized && - (entropy_count >> ENTROPY_SHIFT) > 128) - has_initialized = 1; if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) goto retry; if (has_initialized) { r->initialized = 1; - wake_up_interruptible(&random_read_wait); kill_fasync(&fasync, SIGIO, POLL_IN); } @@ -783,36 +727,13 @@ retry: if (r == &input_pool) { int entropy_bits = entropy_count >> ENTROPY_SHIFT; - struct entropy_store *other = &blocking_pool; if (crng_init < 2) { if (entropy_bits < 128) return; crng_reseed(&primary_crng, r); - entropy_bits = r->entropy_count >> ENTROPY_SHIFT; - } - - /* initialize the blocking pool if necessary */ - if (entropy_bits >= random_read_wakeup_bits && - !other->initialized) { - schedule_work(&other->push_work); - return; - } - - /* should we wake readers? */ - if (entropy_bits >= random_read_wakeup_bits && - wq_has_sleeper(&random_read_wait)) { - wake_up_interruptible(&random_read_wait); - kill_fasync(&fasync, SIGIO, POLL_IN); + entropy_bits = ENTROPY_BITS(r); } - /* If the input pool is getting full, and the blocking - * pool has room, send some entropy to the blocking - * pool. - */ - if (!work_pending(&other->push_work) && - (ENTROPY_BITS(r) > 6 * r->poolinfo->poolbytes) && - (ENTROPY_BITS(other) <= 6 * other->poolinfo->poolbytes)) - schedule_work(&other->push_work); } } @@ -884,7 +805,7 @@ static void crng_initialize(struct crng_state *crng) invalidate_batched_entropy(); numa_crng_init(); crng_init = 2; - pr_notice("random: crng done (trusting CPU's manufacturer)\n"); + pr_notice("crng done (trusting CPU's manufacturer)\n"); } crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; } @@ -946,8 +867,7 @@ static int crng_fast_load(const char *cp, size_t len) if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { invalidate_batched_entropy(); crng_init = 1; - wake_up_interruptible(&crng_init_wait); - pr_notice("random: fast init done\n"); + pr_notice("fast init done\n"); } return 1; } @@ -1032,16 +952,15 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r) crng_init = 2; process_random_ready_list(); wake_up_interruptible(&crng_init_wait); - pr_notice("random: crng init done\n"); + kill_fasync(&fasync, SIGIO, POLL_IN); + pr_notice("crng init done\n"); if (unseeded_warning.missed) { - pr_notice("random: %d get_random_xx warning(s) missed " - "due to ratelimiting\n", + pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n", unseeded_warning.missed); unseeded_warning.missed = 0; } if (urandom_warning.missed) { - pr_notice("random: %d urandom warning(s) missed " - "due to ratelimiting\n", + pr_notice("%d urandom warning(s) missed due to ratelimiting\n", urandom_warning.missed); urandom_warning.missed = 0; } @@ -1246,7 +1165,7 @@ static void add_timer_randomness(struct timer_rand_state *state, unsigned num) /* * delta is now minimum absolute delta. * Round down by 1 bit on general principles, - * and limit entropy entimate to 12 bits. + * and limit entropy estimate to 12 bits. */ credit_entropy_bits(r, min_t(int, fls(delta>>1), 11)); } @@ -1390,57 +1309,6 @@ EXPORT_SYMBOL_GPL(add_disk_randomness); *********************************************************************/ /* - * This utility inline function is responsible for transferring entropy - * from the primary pool to the secondary extraction pool. We make - * sure we pull enough for a 'catastrophic reseed'. - */ -static void _xfer_secondary_pool(struct entropy_store *r, size_t nbytes); -static void xfer_secondary_pool(struct entropy_store *r, size_t nbytes) -{ - if (!r->pull || - r->entropy_count >= (nbytes << (ENTROPY_SHIFT + 3)) || - r->entropy_count > r->poolinfo->poolfracbits) - return; - - _xfer_secondary_pool(r, nbytes); -} - -static void _xfer_secondary_pool(struct entropy_store *r, size_t nbytes) -{ - __u32 tmp[OUTPUT_POOL_WORDS]; - - int bytes = nbytes; - - /* pull at least as much as a wakeup */ - bytes = max_t(int, bytes, random_read_wakeup_bits / 8); - /* but never more than the buffer size */ - bytes = min_t(int, bytes, sizeof(tmp)); - - trace_xfer_secondary_pool(r->name, bytes * 8, nbytes * 8, - ENTROPY_BITS(r), ENTROPY_BITS(r->pull)); - bytes = extract_entropy(r->pull, tmp, bytes, - random_read_wakeup_bits / 8, 0); - mix_pool_bytes(r, tmp, bytes); - credit_entropy_bits(r, bytes*8); -} - -/* - * Used as a workqueue function so that when the input pool is getting - * full, we can "spill over" some entropy to the output pools. That - * way the output pools can store some of the excess entropy instead - * of letting it go to waste. - */ -static void push_to_pool(struct work_struct *work) -{ - struct entropy_store *r = container_of(work, struct entropy_store, - push_work); - BUG_ON(!r); - _xfer_secondary_pool(r, random_read_wakeup_bits/8); - trace_push_to_pool(r->name, r->entropy_count >> ENTROPY_SHIFT, - r->pull->entropy_count >> ENTROPY_SHIFT); -} - -/* * This function decides how many bytes to actually take from the * given pool, and also debits the entropy count accordingly. */ @@ -1465,10 +1333,9 @@ retry: if (ibytes < min) ibytes = 0; - if (unlikely(entropy_count < 0)) { - pr_warn("random: negative entropy count: pool %s count %d\n", + if (WARN_ON(entropy_count < 0)) { + pr_warn("negative entropy count: pool %s count %d\n", r->name, entropy_count); - WARN_ON(1); entropy_count = 0; } nfrac = ibytes << (ENTROPY_SHIFT + 3); @@ -1481,8 +1348,7 @@ retry: goto retry; trace_debit_entropy(r->name, 8 * ibytes); - if (ibytes && - (r->entropy_count >> ENTROPY_SHIFT) < random_write_wakeup_bits) { + if (ibytes && ENTROPY_BITS(r) < random_write_wakeup_bits) { wake_up_interruptible(&random_write_wait); kill_fasync(&fasync, SIGIO, POLL_OUT); } @@ -1603,7 +1469,6 @@ static ssize_t extract_entropy(struct entropy_store *r, void *buf, spin_unlock_irqrestore(&r->lock, flags); trace_extract_entropy(r->name, EXTRACT_SIZE, ENTROPY_BITS(r), _RET_IP_); - xfer_secondary_pool(r, EXTRACT_SIZE); extract_buf(r, tmp); spin_lock_irqsave(&r->lock, flags); memcpy(r->last_data, tmp, EXTRACT_SIZE); @@ -1612,60 +1477,11 @@ static ssize_t extract_entropy(struct entropy_store *r, void *buf, } trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_); - xfer_secondary_pool(r, nbytes); nbytes = account(r, nbytes, min, reserved); return _extract_entropy(r, buf, nbytes, fips_enabled); } -/* - * This function extracts randomness from the "entropy pool", and - * returns it in a userspace buffer. - */ -static ssize_t extract_entropy_user(struct entropy_store *r, void __user *buf, - size_t nbytes) -{ - ssize_t ret = 0, i; - __u8 tmp[EXTRACT_SIZE]; - int large_request = (nbytes > 256); - - trace_extract_entropy_user(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_); - if (!r->initialized && r->pull) { - xfer_secondary_pool(r, ENTROPY_BITS(r->pull)/8); - if (!r->initialized) - return 0; - } - xfer_secondary_pool(r, nbytes); - nbytes = account(r, nbytes, 0, 0); - - while (nbytes) { - if (large_request && need_resched()) { - if (signal_pending(current)) { - if (ret == 0) - ret = -ERESTARTSYS; - break; - } - schedule(); - } - - extract_buf(r, tmp); - i = min_t(int, nbytes, EXTRACT_SIZE); - if (copy_to_user(buf, tmp, i)) { - ret = -EFAULT; - break; - } - - nbytes -= i; - buf += i; - ret += i; - } - - /* Wipe data just returned from memory */ - memzero_explicit(tmp, sizeof(tmp)); - - return ret; -} - #define warn_unseeded_randomness(previous) \ _warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous)) @@ -1687,8 +1503,9 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller, print_once = true; #endif if (__ratelimit(&unseeded_warning)) - pr_notice("random: %s called from %pS with crng_init=%d\n", - func_name, caller, crng_init); + printk_deferred(KERN_NOTICE "random: %s called from %pS " + "with crng_init=%d\n", func_name, caller, + crng_init); } /* @@ -1931,7 +1748,6 @@ static void __init init_std_data(struct entropy_store *r) ktime_t now = ktime_get_real(); unsigned long rv; - r->last_pulled = jiffies; mix_pool_bytes(r, &now, sizeof(now)); for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) { if (!arch_get_random_seed_long(&rv) && @@ -1955,7 +1771,6 @@ static void __init init_std_data(struct entropy_store *r) int __init rand_initialize(void) { init_std_data(&input_pool); - init_std_data(&blocking_pool); crng_initialize(&primary_crng); crng_global_init_time = jiffies; if (ratelimit_disable) { @@ -1983,40 +1798,15 @@ void rand_initialize_disk(struct gendisk *disk) #endif static ssize_t -_random_read(int nonblock, char __user *buf, size_t nbytes) +urandom_read_nowarn(struct file *file, char __user *buf, size_t nbytes, + loff_t *ppos) { - ssize_t n; - - if (nbytes == 0) - return 0; - - nbytes = min_t(size_t, nbytes, SEC_XFER_SIZE); - while (1) { - n = extract_entropy_user(&blocking_pool, buf, nbytes); - if (n < 0) - return n; - trace_random_read(n*8, (nbytes-n)*8, - ENTROPY_BITS(&blocking_pool), - ENTROPY_BITS(&input_pool)); - if (n > 0) - return n; - - /* Pool is (near) empty. Maybe wait and retry. */ - if (nonblock) - return -EAGAIN; - - wait_event_interruptible(random_read_wait, - blocking_pool.initialized && - (ENTROPY_BITS(&input_pool) >= random_read_wakeup_bits)); - if (signal_pending(current)) - return -ERESTARTSYS; - } -} + int ret; -static ssize_t -random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) -{ - return _random_read(file->f_flags & O_NONBLOCK, buf, nbytes); + nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); + ret = extract_crng_user(buf, nbytes); + trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool)); + return ret; } static ssize_t @@ -2024,22 +1814,29 @@ urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { unsigned long flags; static int maxwarn = 10; - int ret; if (!crng_ready() && maxwarn > 0) { maxwarn--; if (__ratelimit(&urandom_warning)) - printk(KERN_NOTICE "random: %s: uninitialized " - "urandom read (%zd bytes read)\n", - current->comm, nbytes); + pr_notice("%s: uninitialized urandom read (%zd bytes read)\n", + current->comm, nbytes); spin_lock_irqsave(&primary_crng.lock, flags); crng_init_cnt = 0; spin_unlock_irqrestore(&primary_crng.lock, flags); } - nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); - ret = extract_crng_user(buf, nbytes); - trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool)); - return ret; + + return urandom_read_nowarn(file, buf, nbytes, ppos); +} + +static ssize_t +random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) +{ + int ret; + + ret = wait_for_random_bytes(); + if (ret != 0) + return ret; + return urandom_read_nowarn(file, buf, nbytes, ppos); } static __poll_t @@ -2047,10 +1844,10 @@ random_poll(struct file *file, poll_table * wait) { __poll_t mask; - poll_wait(file, &random_read_wait, wait); + poll_wait(file, &crng_init_wait, wait); poll_wait(file, &random_write_wait, wait); mask = 0; - if (ENTROPY_BITS(&input_pool) >= random_read_wakeup_bits) + if (crng_ready()) mask |= EPOLLIN | EPOLLRDNORM; if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits) mask |= EPOLLOUT | EPOLLWRNORM; @@ -2141,7 +1938,6 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) if (!capable(CAP_SYS_ADMIN)) return -EPERM; input_pool.entropy_count = 0; - blocking_pool.entropy_count = 0; return 0; case RNDRESEEDCRNG: if (!capable(CAP_SYS_ADMIN)) @@ -2185,23 +1981,27 @@ SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, { int ret; - if (flags & ~(GRND_NONBLOCK|GRND_RANDOM)) + if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE)) + return -EINVAL; + + /* + * Requesting insecure and blocking randomness at the same time makes + * no sense. + */ + if ((flags & (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM)) return -EINVAL; if (count > INT_MAX) count = INT_MAX; - if (flags & GRND_RANDOM) - return _random_read(flags & GRND_NONBLOCK, buf, count); - - if (!crng_ready()) { + if (!(flags & GRND_INSECURE) && !crng_ready()) { if (flags & GRND_NONBLOCK) return -EAGAIN; ret = wait_for_random_bytes(); if (unlikely(ret)) return ret; } - return urandom_read(NULL, buf, count, NULL); + return urandom_read_nowarn(NULL, buf, count, NULL); } /******************************************************************** @@ -2214,8 +2014,7 @@ SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, #include <linux/sysctl.h> -static int min_read_thresh = 8, min_write_thresh; -static int max_read_thresh = OUTPUT_POOL_WORDS * 32; +static int min_write_thresh; static int max_write_thresh = INPUT_POOL_WORDS * 32; static int random_min_urandom_seed = 60; static char sysctl_bootid[16]; @@ -2291,15 +2090,6 @@ struct ctl_table random_table[] = { .data = &input_pool.entropy_count, }, { - .procname = "read_wakeup_threshold", - .data = &random_read_wakeup_bits, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_read_thresh, - .extra2 = &max_read_thresh, - }, - { .procname = "write_wakeup_threshold", .data = &random_write_wakeup_bits, .maxlen = sizeof(int), diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 45653a0e6ecd..bcb257baed06 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -27,7 +27,7 @@ config COMMON_CLK_WM831X tristate "Clock driver for WM831x/2x PMICs" depends on MFD_WM831X ---help--- - Supports the clocking subsystem of the WM831x/2x series of + Supports the clocking subsystem of the WM831x/2x series of PMICs from Wolfson Microelectronics. source "drivers/clk/versatile/Kconfig" @@ -174,6 +174,18 @@ config COMMON_CLK_CS2000_CP help If you say yes here you get support for the CS2000 clock multiplier. +config COMMON_CLK_FSL_SAI + bool "Clock driver for BCLK of Freescale SAI cores" + depends on ARCH_LAYERSCAPE || COMPILE_TEST + help + This driver supports the Freescale SAI (Synchronous Audio Interface) + to be used as a generic clock output. Some SoCs have restrictions + regarding the possible pin multiplexer settings. Eg. on some SoCs + two SAI interfaces can only be enabled together. If just one is + needed, the BCLK pin of the second one can be used as general + purpose clock output. Ideally, it can be used to drive an audio + codec (sometimes known as MCLK). + config COMMON_CLK_GEMINI bool "Clock driver for Cortina Systems Gemini SoC" depends on ARCH_GEMINI || COMPILE_TEST @@ -225,6 +237,16 @@ config CLK_QORIQ This adds the clock driver support for Freescale QorIQ platforms using common clock framework. +config CLK_LS1028A_PLLDIG + tristate "Clock driver for LS1028A Display output" + depends on ARCH_LAYERSCAPE || COMPILE_TEST + default ARCH_LAYERSCAPE + help + This driver support the Display output interfaces(LCD, DPHY) pixel clocks + of the QorIQ Layerscape LS1028A, as implemented TSMC CLN28HPM PLL. Not all + features of the PLL are currently supported by the driver. By default, + configured bypass mode with this PLL. + config COMMON_CLK_XGENE bool "Clock driver for APM XGene SoC" default ARCH_XGENE @@ -305,10 +327,10 @@ config COMMON_CLK_MMP2 Support for Marvell MMP2 and MMP3 SoC clocks config COMMON_CLK_BD718XX - tristate "Clock driver for ROHM BD718x7 PMIC" - depends on MFD_ROHM_BD718XX || MFD_ROHM_BD70528 + tristate "Clock driver for 32K clk gates on ROHM PMICs" + depends on MFD_ROHM_BD718XX || MFD_ROHM_BD70528 || MFD_ROHM_BD71828 help - This driver supports ROHM BD71837, ROHM BD71847 and + This driver supports ROHM BD71837, ROHM BD71847, ROHM BD71828 and ROHM BD70528 PMICs clock gates. config COMMON_CLK_FIXED_MMIO diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 0696a0c1ab58..f4169cc2fd31 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_ARCH_CLPS711X) += clk-clps711x.o obj-$(CONFIG_COMMON_CLK_CS2000_CP) += clk-cs2000-cp.o obj-$(CONFIG_ARCH_EFM32) += clk-efm32gg.o obj-$(CONFIG_COMMON_CLK_FIXED_MMIO) += clk-fixed-mmio.o +obj-$(CONFIG_COMMON_CLK_FSL_SAI) += clk-fsl-sai.o obj-$(CONFIG_COMMON_CLK_GEMINI) += clk-gemini.o obj-$(CONFIG_COMMON_CLK_ASPEED) += clk-aspeed.o obj-$(CONFIG_MACH_ASPEED_G6) += clk-ast2600.o @@ -44,6 +45,7 @@ obj-$(CONFIG_ARCH_NPCM7XX) += clk-npcm7xx.o obj-$(CONFIG_ARCH_NSPIRE) += clk-nspire.o obj-$(CONFIG_COMMON_CLK_OXNAS) += clk-oxnas.o obj-$(CONFIG_COMMON_CLK_PALMAS) += clk-palmas.o +obj-$(CONFIG_CLK_LS1028A_PLLDIG) += clk-plldig.o obj-$(CONFIG_COMMON_CLK_PWM) += clk-pwm.o obj-$(CONFIG_CLK_QORIQ) += clk-qoriq.o obj-$(CONFIG_COMMON_CLK_RK808) += clk-rk808.o diff --git a/drivers/clk/at91/clk-sam9x60-pll.c b/drivers/clk/at91/clk-sam9x60-pll.c index 34b817825b22..dfb354a5ff18 100644 --- a/drivers/clk/at91/clk-sam9x60-pll.c +++ b/drivers/clk/at91/clk-sam9x60-pll.c @@ -25,7 +25,8 @@ #define PMC_PLL_CTRL1_MUL_MSK GENMASK(30, 24) #define PMC_PLL_ACR 0x18 -#define PMC_PLL_ACR_DEFAULT 0x1b040010UL +#define PMC_PLL_ACR_DEFAULT_UPLL 0x12020010UL +#define PMC_PLL_ACR_DEFAULT_PLLA 0x00020010UL #define PMC_PLL_ACR_UTMIVR BIT(12) #define PMC_PLL_ACR_UTMIBG BIT(13) #define PMC_PLL_ACR_LOOP_FILTER_MSK GENMASK(31, 24) @@ -88,7 +89,10 @@ static int sam9x60_pll_prepare(struct clk_hw *hw) } /* Recommended value for PMC_PLL_ACR */ - val = PMC_PLL_ACR_DEFAULT; + if (pll->characteristics->upll) + val = PMC_PLL_ACR_DEFAULT_UPLL; + else + val = PMC_PLL_ACR_DEFAULT_PLLA; regmap_write(regmap, PMC_PLL_ACR, val); regmap_write(regmap, PMC_PLL_CTRL1, diff --git a/drivers/clk/at91/sam9x60.c b/drivers/clk/at91/sam9x60.c index 86238d5ecb4d..77398aefeb6d 100644 --- a/drivers/clk/at91/sam9x60.c +++ b/drivers/clk/at91/sam9x60.c @@ -47,6 +47,7 @@ static const struct clk_programmable_layout sam9x60_programmable_layout = { .pres_shift = 8, .css_mask = 0x1f, .have_slck_mck = 0, + .is_pres_direct = 1, }; static const struct clk_pcr_layout sam9x60_pcr_layout = { diff --git a/drivers/clk/clk-asm9260.c b/drivers/clk/clk-asm9260.c index dd0f90c9dd0e..536b59aabd2c 100644 --- a/drivers/clk/clk-asm9260.c +++ b/drivers/clk/clk-asm9260.c @@ -260,7 +260,6 @@ static void __init asm9260_acc_init(struct device_node *np) const char *ref_clk, *pll_clk = "pll"; u32 rate; int n; - u32 accuracy = 0; clk_data = kzalloc(struct_size(clk_data, hws, MAX_CLKS), GFP_KERNEL); if (!clk_data) @@ -275,10 +274,11 @@ static void __init asm9260_acc_init(struct device_node *np) /* register pll */ rate = (ioread32(base + HW_SYSPLLCTRL) & 0xffff) * 1000000; + /* TODO: Convert to DT parent scheme */ ref_clk = of_clk_get_parent_name(np, 0); - accuracy = clk_get_accuracy(__clk_lookup(ref_clk)); - hw = clk_hw_register_fixed_rate_with_accuracy(NULL, pll_clk, - ref_clk, 0, rate, accuracy); + hw = __clk_hw_register_fixed_rate_with_accuracy(NULL, NULL, pll_clk, + ref_clk, NULL, NULL, 0, rate, 0, + CLK_FIXED_RATE_PARENT_ACCURACY); if (IS_ERR(hw)) panic("%pOFn: can't register REFCLK. Check DT!", np); diff --git a/drivers/clk/clk-bd718x7.c b/drivers/clk/clk-bd718x7.c index 00926c587390..b52e8d6f660c 100644 --- a/drivers/clk/clk-bd718x7.c +++ b/drivers/clk/clk-bd718x7.c @@ -7,12 +7,25 @@ #include <linux/err.h> #include <linux/platform_device.h> #include <linux/slab.h> -#include <linux/mfd/rohm-bd718x7.h> -#include <linux/mfd/rohm-bd70528.h> +#include <linux/mfd/rohm-generic.h> #include <linux/clk-provider.h> #include <linux/clkdev.h> #include <linux/regmap.h> +/* clk control registers */ +/* BD70528 */ +#define BD70528_REG_OUT32K 0x2c +/* BD71828 */ +#define BD71828_REG_OUT32K 0x4B +/* BD71837 and BD71847 */ +#define BD718XX_REG_OUT32K 0x2E + +/* + * BD71837, BD71847, BD70528 and BD71828 all use bit [0] to clk output control + */ +#define CLK_OUT_EN_MASK BIT(0) + + struct bd718xx_clk { struct clk_hw hw; u8 reg; @@ -21,10 +34,8 @@ struct bd718xx_clk { struct rohm_regmap_dev *mfd; }; -static int bd71837_clk_set(struct clk_hw *hw, int status) +static int bd71837_clk_set(struct bd718xx_clk *c, unsigned int status) { - struct bd718xx_clk *c = container_of(hw, struct bd718xx_clk, hw); - return regmap_update_bits(c->mfd->regmap, c->reg, c->mask, status); } @@ -33,14 +44,16 @@ static void bd71837_clk_disable(struct clk_hw *hw) int rv; struct bd718xx_clk *c = container_of(hw, struct bd718xx_clk, hw); - rv = bd71837_clk_set(hw, 0); + rv = bd71837_clk_set(c, 0); if (rv) dev_dbg(&c->pdev->dev, "Failed to disable 32K clk (%d)\n", rv); } static int bd71837_clk_enable(struct clk_hw *hw) { - return bd71837_clk_set(hw, 1); + struct bd718xx_clk *c = container_of(hw, struct bd718xx_clk, hw); + + return bd71837_clk_set(c, 0xffffffff); } static int bd71837_clk_is_enabled(struct clk_hw *hw) @@ -74,6 +87,7 @@ static int bd71837_clk_probe(struct platform_device *pdev) .name = "bd718xx-32k-out", .ops = &bd71837_clk_ops, }; + enum rohm_chip_type chip = platform_get_device_id(pdev)->driver_data; c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); if (!c) @@ -87,15 +101,19 @@ static int bd71837_clk_probe(struct platform_device *pdev) dev_err(&pdev->dev, "No parent clk found\n"); return -EINVAL; } - switch (mfd->chip_type) { + switch (chip) { case ROHM_CHIP_TYPE_BD71837: case ROHM_CHIP_TYPE_BD71847: c->reg = BD718XX_REG_OUT32K; - c->mask = BD718XX_OUT32K_EN; + c->mask = CLK_OUT_EN_MASK; + break; + case ROHM_CHIP_TYPE_BD71828: + c->reg = BD71828_REG_OUT32K; + c->mask = CLK_OUT_EN_MASK; break; case ROHM_CHIP_TYPE_BD70528: - c->reg = BD70528_REG_CLK_OUT; - c->mask = BD70528_CLK_OUT_EN_MASK; + c->reg = BD70528_REG_OUT32K; + c->mask = CLK_OUT_EN_MASK; break; default: dev_err(&pdev->dev, "Unknown clk chip\n"); @@ -121,11 +139,21 @@ static int bd71837_clk_probe(struct platform_device *pdev) return rval; } +static const struct platform_device_id bd718x7_clk_id[] = { + { "bd71837-clk", ROHM_CHIP_TYPE_BD71837 }, + { "bd71847-clk", ROHM_CHIP_TYPE_BD71847 }, + { "bd70528-clk", ROHM_CHIP_TYPE_BD70528 }, + { "bd71828-clk", ROHM_CHIP_TYPE_BD71828 }, + { }, +}; +MODULE_DEVICE_TABLE(platform, bd718x7_clk_id); + static struct platform_driver bd71837_clk = { .driver = { .name = "bd718xx-clk", }, .probe = bd71837_clk_probe, + .id_table = bd718x7_clk_id, }; module_platform_driver(bd71837_clk); diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c index 4cd175afce9b..e6d6599d310a 100644 --- a/drivers/clk/clk-bm1880.c +++ b/drivers/clk/clk-bm1880.c @@ -474,11 +474,10 @@ static struct bm1880_composite_clock bm1880_composite_clks[] = { static unsigned long bm1880_pll_rate_calc(u32 regval, unsigned long parent_rate) { u64 numerator; - u32 fbdiv, fref, refdiv; + u32 fbdiv, refdiv; u32 postdiv1, postdiv2, denominator; fbdiv = (regval >> 16) & 0xfff; - fref = parent_rate; refdiv = regval & 0x1f; postdiv1 = (regval >> 8) & 0x7; postdiv2 = (regval >> 12) & 0x7; diff --git a/drivers/clk/clk-composite.c b/drivers/clk/clk-composite.c index 3e9c3e608769..7376f573bfdb 100644 --- a/drivers/clk/clk-composite.c +++ b/drivers/clk/clk-composite.c @@ -199,8 +199,9 @@ static void clk_composite_disable(struct clk_hw *hw) gate_ops->disable(gate_hw); } -struct clk_hw *clk_hw_register_composite(struct device *dev, const char *name, - const char * const *parent_names, int num_parents, +static struct clk_hw *__clk_hw_register_composite(struct device *dev, + const char *name, const char * const *parent_names, + const struct clk_parent_data *pdata, int num_parents, struct clk_hw *mux_hw, const struct clk_ops *mux_ops, struct clk_hw *rate_hw, const struct clk_ops *rate_ops, struct clk_hw *gate_hw, const struct clk_ops *gate_ops, @@ -218,7 +219,10 @@ struct clk_hw *clk_hw_register_composite(struct device *dev, const char *name, init.name = name; init.flags = flags; - init.parent_names = parent_names; + if (parent_names) + init.parent_names = parent_names; + else + init.parent_data = pdata; init.num_parents = num_parents; hw = &composite->hw; @@ -312,6 +316,34 @@ err: return hw; } +struct clk_hw *clk_hw_register_composite(struct device *dev, const char *name, + const char * const *parent_names, int num_parents, + struct clk_hw *mux_hw, const struct clk_ops *mux_ops, + struct clk_hw *rate_hw, const struct clk_ops *rate_ops, + struct clk_hw *gate_hw, const struct clk_ops *gate_ops, + unsigned long flags) +{ + return __clk_hw_register_composite(dev, name, parent_names, NULL, + num_parents, mux_hw, mux_ops, + rate_hw, rate_ops, gate_hw, + gate_ops, flags); +} + +struct clk_hw *clk_hw_register_composite_pdata(struct device *dev, + const char *name, + const struct clk_parent_data *parent_data, + int num_parents, + struct clk_hw *mux_hw, const struct clk_ops *mux_ops, + struct clk_hw *rate_hw, const struct clk_ops *rate_ops, + struct clk_hw *gate_hw, const struct clk_ops *gate_ops, + unsigned long flags) +{ + return __clk_hw_register_composite(dev, name, NULL, parent_data, + num_parents, mux_hw, mux_ops, + rate_hw, rate_ops, gate_hw, + gate_ops, flags); +} + struct clk *clk_register_composite(struct device *dev, const char *name, const char * const *parent_names, int num_parents, struct clk_hw *mux_hw, const struct clk_ops *mux_ops, @@ -329,6 +361,24 @@ struct clk *clk_register_composite(struct device *dev, const char *name, return hw->clk; } +struct clk *clk_register_composite_pdata(struct device *dev, const char *name, + const struct clk_parent_data *parent_data, + int num_parents, + struct clk_hw *mux_hw, const struct clk_ops *mux_ops, + struct clk_hw *rate_hw, const struct clk_ops *rate_ops, + struct clk_hw *gate_hw, const struct clk_ops *gate_ops, + unsigned long flags) +{ + struct clk_hw *hw; + + hw = clk_hw_register_composite_pdata(dev, name, parent_data, + num_parents, mux_hw, mux_ops, rate_hw, rate_ops, + gate_hw, gate_ops, flags); + if (IS_ERR(hw)) + return ERR_CAST(hw); + return hw->clk; +} + void clk_unregister_composite(struct clk *clk) { struct clk_composite *composite; diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c index 098b2b01f0af..8de12cb0c43d 100644 --- a/drivers/clk/clk-divider.c +++ b/drivers/clk/clk-divider.c @@ -463,11 +463,12 @@ const struct clk_ops clk_divider_ro_ops = { }; EXPORT_SYMBOL_GPL(clk_divider_ro_ops); -static struct clk_hw *_register_divider(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, - void __iomem *reg, u8 shift, u8 width, - u8 clk_divider_flags, const struct clk_div_table *table, - spinlock_t *lock) +struct clk_hw *__clk_hw_register_divider(struct device *dev, + struct device_node *np, const char *name, + const char *parent_name, const struct clk_hw *parent_hw, + const struct clk_parent_data *parent_data, unsigned long flags, + void __iomem *reg, u8 shift, u8 width, u8 clk_divider_flags, + const struct clk_div_table *table, spinlock_t *lock) { struct clk_divider *div; struct clk_hw *hw; @@ -514,55 +515,7 @@ static struct clk_hw *_register_divider(struct device *dev, const char *name, return hw; } - -/** - * clk_register_divider - register a divider clock with the clock framework - * @dev: device registering this clock - * @name: name of this clock - * @parent_name: name of clock's parent - * @flags: framework-specific flags - * @reg: register address to adjust divider - * @shift: number of bits to shift the bitfield - * @width: width of the bitfield - * @clk_divider_flags: divider-specific flags for this clock - * @lock: shared register lock for this clock - */ -struct clk *clk_register_divider(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, - void __iomem *reg, u8 shift, u8 width, - u8 clk_divider_flags, spinlock_t *lock) -{ - struct clk_hw *hw; - - hw = _register_divider(dev, name, parent_name, flags, reg, shift, - width, clk_divider_flags, NULL, lock); - if (IS_ERR(hw)) - return ERR_CAST(hw); - return hw->clk; -} -EXPORT_SYMBOL_GPL(clk_register_divider); - -/** - * clk_hw_register_divider - register a divider clock with the clock framework - * @dev: device registering this clock - * @name: name of this clock - * @parent_name: name of clock's parent - * @flags: framework-specific flags - * @reg: register address to adjust divider - * @shift: number of bits to shift the bitfield - * @width: width of the bitfield - * @clk_divider_flags: divider-specific flags for this clock - * @lock: shared register lock for this clock - */ -struct clk_hw *clk_hw_register_divider(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, - void __iomem *reg, u8 shift, u8 width, - u8 clk_divider_flags, spinlock_t *lock) -{ - return _register_divider(dev, name, parent_name, flags, reg, shift, - width, clk_divider_flags, NULL, lock); -} -EXPORT_SYMBOL_GPL(clk_hw_register_divider); +EXPORT_SYMBOL_GPL(__clk_hw_register_divider); /** * clk_register_divider_table - register a table based divider clock with @@ -586,39 +539,15 @@ struct clk *clk_register_divider_table(struct device *dev, const char *name, { struct clk_hw *hw; - hw = _register_divider(dev, name, parent_name, flags, reg, shift, - width, clk_divider_flags, table, lock); + hw = __clk_hw_register_divider(dev, NULL, name, parent_name, NULL, + NULL, flags, reg, shift, width, clk_divider_flags, + table, lock); if (IS_ERR(hw)) return ERR_CAST(hw); return hw->clk; } EXPORT_SYMBOL_GPL(clk_register_divider_table); -/** - * clk_hw_register_divider_table - register a table based divider clock with - * the clock framework - * @dev: device registering this clock - * @name: name of this clock - * @parent_name: name of clock's parent - * @flags: framework-specific flags - * @reg: register address to adjust divider - * @shift: number of bits to shift the bitfield - * @width: width of the bitfield - * @clk_divider_flags: divider-specific flags for this clock - * @table: array of divider/value pairs ending with a div set to 0 - * @lock: shared register lock for this clock - */ -struct clk_hw *clk_hw_register_divider_table(struct device *dev, - const char *name, const char *parent_name, unsigned long flags, - void __iomem *reg, u8 shift, u8 width, - u8 clk_divider_flags, const struct clk_div_table *table, - spinlock_t *lock) -{ - return _register_divider(dev, name, parent_name, flags, reg, shift, - width, clk_divider_flags, table, lock); -} -EXPORT_SYMBOL_GPL(clk_hw_register_divider_table); - void clk_unregister_divider(struct clk *clk) { struct clk_divider *div; diff --git a/drivers/clk/clk-fixed-rate.c b/drivers/clk/clk-fixed-rate.c index 2c4486c09040..77499a27c8fb 100644 --- a/drivers/clk/clk-fixed-rate.c +++ b/drivers/clk/clk-fixed-rate.c @@ -24,6 +24,8 @@ * parent - fixed parent. No clk_set_parent support */ +#define to_clk_fixed_rate(_hw) container_of(_hw, struct clk_fixed_rate, hw) + static unsigned long clk_fixed_rate_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { @@ -33,7 +35,12 @@ static unsigned long clk_fixed_rate_recalc_rate(struct clk_hw *hw, static unsigned long clk_fixed_rate_recalc_accuracy(struct clk_hw *hw, unsigned long parent_accuracy) { - return to_clk_fixed_rate(hw)->fixed_accuracy; + struct clk_fixed_rate *fixed = to_clk_fixed_rate(hw); + + if (fixed->flags & CLK_FIXED_RATE_PARENT_ACCURACY) + return parent_accuracy; + + return fixed->fixed_accuracy; } const struct clk_ops clk_fixed_rate_ops = { @@ -42,24 +49,17 @@ const struct clk_ops clk_fixed_rate_ops = { }; EXPORT_SYMBOL_GPL(clk_fixed_rate_ops); -/** - * clk_hw_register_fixed_rate_with_accuracy - register fixed-rate clock with - * the clock framework - * @dev: device that is registering this clock - * @name: name of this clock - * @parent_name: name of clock's parent - * @flags: framework-specific flags - * @fixed_rate: non-adjustable clock rate - * @fixed_accuracy: non-adjustable clock rate - */ -struct clk_hw *clk_hw_register_fixed_rate_with_accuracy(struct device *dev, - const char *name, const char *parent_name, unsigned long flags, - unsigned long fixed_rate, unsigned long fixed_accuracy) +struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, + struct device_node *np, const char *name, + const char *parent_name, const struct clk_hw *parent_hw, + const struct clk_parent_data *parent_data, unsigned long flags, + unsigned long fixed_rate, unsigned long fixed_accuracy, + unsigned long clk_fixed_flags) { struct clk_fixed_rate *fixed; struct clk_hw *hw; struct clk_init_data init = {}; - int ret; + int ret = -EINVAL; /* allocate fixed-rate clock */ fixed = kzalloc(sizeof(*fixed), GFP_KERNEL); @@ -69,17 +69,26 @@ struct clk_hw *clk_hw_register_fixed_rate_with_accuracy(struct device *dev, init.name = name; init.ops = &clk_fixed_rate_ops; init.flags = flags; - init.parent_names = (parent_name ? &parent_name: NULL); - init.num_parents = (parent_name ? 1 : 0); + init.parent_names = parent_name ? &parent_name : NULL; + init.parent_hws = parent_hw ? &parent_hw : NULL; + init.parent_data = parent_data; + if (parent_name || parent_hw || parent_data) + init.num_parents = 1; + else + init.num_parents = 0; /* struct clk_fixed_rate assignments */ + fixed->flags = clk_fixed_flags; fixed->fixed_rate = fixed_rate; fixed->fixed_accuracy = fixed_accuracy; fixed->hw.init = &init; /* register the clock */ hw = &fixed->hw; - ret = clk_hw_register(dev, hw); + if (dev || !np) + ret = clk_hw_register(dev, hw); + else if (np) + ret = of_clk_hw_register(np, hw); if (ret) { kfree(fixed); hw = ERR_PTR(ret); @@ -87,47 +96,20 @@ struct clk_hw *clk_hw_register_fixed_rate_with_accuracy(struct device *dev, return hw; } -EXPORT_SYMBOL_GPL(clk_hw_register_fixed_rate_with_accuracy); +EXPORT_SYMBOL_GPL(__clk_hw_register_fixed_rate); -struct clk *clk_register_fixed_rate_with_accuracy(struct device *dev, - const char *name, const char *parent_name, unsigned long flags, - unsigned long fixed_rate, unsigned long fixed_accuracy) +struct clk *clk_register_fixed_rate(struct device *dev, const char *name, + const char *parent_name, unsigned long flags, + unsigned long fixed_rate) { struct clk_hw *hw; hw = clk_hw_register_fixed_rate_with_accuracy(dev, name, parent_name, - flags, fixed_rate, fixed_accuracy); + flags, fixed_rate, 0); if (IS_ERR(hw)) return ERR_CAST(hw); return hw->clk; } -EXPORT_SYMBOL_GPL(clk_register_fixed_rate_with_accuracy); - -/** - * clk_hw_register_fixed_rate - register fixed-rate clock with the clock - * framework - * @dev: device that is registering this clock - * @name: name of this clock - * @parent_name: name of clock's parent - * @flags: framework-specific flags - * @fixed_rate: non-adjustable clock rate - */ -struct clk_hw *clk_hw_register_fixed_rate(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, - unsigned long fixed_rate) -{ - return clk_hw_register_fixed_rate_with_accuracy(dev, name, parent_name, - flags, fixed_rate, 0); -} -EXPORT_SYMBOL_GPL(clk_hw_register_fixed_rate); - -struct clk *clk_register_fixed_rate(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, - unsigned long fixed_rate) -{ - return clk_register_fixed_rate_with_accuracy(dev, name, parent_name, - flags, fixed_rate, 0); -} EXPORT_SYMBOL_GPL(clk_register_fixed_rate); void clk_unregister_fixed_rate(struct clk *clk) @@ -155,9 +137,9 @@ void clk_hw_unregister_fixed_rate(struct clk_hw *hw) EXPORT_SYMBOL_GPL(clk_hw_unregister_fixed_rate); #ifdef CONFIG_OF -static struct clk *_of_fixed_clk_setup(struct device_node *node) +static struct clk_hw *_of_fixed_clk_setup(struct device_node *node) { - struct clk *clk; + struct clk_hw *hw; const char *clk_name = node->name; u32 rate; u32 accuracy = 0; @@ -170,18 +152,18 @@ static struct clk *_of_fixed_clk_setup(struct device_node *node) of_property_read_string(node, "clock-output-names", &clk_name); - clk = clk_register_fixed_rate_with_accuracy(NULL, clk_name, NULL, + hw = clk_hw_register_fixed_rate_with_accuracy(NULL, clk_name, NULL, 0, rate, accuracy); - if (IS_ERR(clk)) - return clk; + if (IS_ERR(hw)) + return hw; - ret = of_clk_add_provider(node, of_clk_src_simple_get, clk); + ret = of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw); if (ret) { - clk_unregister(clk); + clk_hw_unregister_fixed_rate(hw); return ERR_PTR(ret); } - return clk; + return hw; } /** @@ -195,27 +177,27 @@ CLK_OF_DECLARE(fixed_clk, "fixed-clock", of_fixed_clk_setup); static int of_fixed_clk_remove(struct platform_device *pdev) { - struct clk *clk = platform_get_drvdata(pdev); + struct clk_hw *hw = platform_get_drvdata(pdev); of_clk_del_provider(pdev->dev.of_node); - clk_unregister_fixed_rate(clk); + clk_hw_unregister_fixed_rate(hw); return 0; } static int of_fixed_clk_probe(struct platform_device *pdev) { - struct clk *clk; + struct clk_hw *hw; /* * This function is not executed when of_fixed_clk_setup * succeeded. */ - clk = _of_fixed_clk_setup(pdev->dev.of_node); - if (IS_ERR(clk)) - return PTR_ERR(clk); + hw = _of_fixed_clk_setup(pdev->dev.of_node); + if (IS_ERR(hw)) + return PTR_ERR(hw); - platform_set_drvdata(pdev, clk); + platform_set_drvdata(pdev, hw); return 0; } @@ -224,7 +206,6 @@ static const struct of_device_id of_fixed_clk_ids[] = { { .compatible = "fixed-clock" }, { } }; -MODULE_DEVICE_TABLE(of, of_fixed_clk_ids); static struct platform_driver of_fixed_clk_driver = { .driver = { diff --git a/drivers/clk/clk-fsl-sai.c b/drivers/clk/clk-fsl-sai.c new file mode 100644 index 000000000000..0221180a4dd7 --- /dev/null +++ b/drivers/clk/clk-fsl-sai.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Freescale SAI BCLK as a generic clock driver + * + * Copyright 2020 Michael Walle <michael@walle.cc> + */ + +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/clk-provider.h> +#include <linux/err.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/slab.h> + +#define I2S_CSR 0x00 +#define I2S_CR2 0x08 +#define CSR_BCE_BIT 28 +#define CR2_BCD BIT(24) +#define CR2_DIV_SHIFT 0 +#define CR2_DIV_WIDTH 8 + +struct fsl_sai_clk { + struct clk_divider div; + struct clk_gate gate; + spinlock_t lock; +}; + +static int fsl_sai_clk_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct fsl_sai_clk *sai_clk; + struct clk_parent_data pdata = { .index = 0 }; + void __iomem *base; + struct clk_hw *hw; + struct resource *res; + + sai_clk = devm_kzalloc(dev, sizeof(*sai_clk), GFP_KERNEL); + if (!sai_clk) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + spin_lock_init(&sai_clk->lock); + + sai_clk->gate.reg = base + I2S_CSR; + sai_clk->gate.bit_idx = CSR_BCE_BIT; + sai_clk->gate.lock = &sai_clk->lock; + + sai_clk->div.reg = base + I2S_CR2; + sai_clk->div.shift = CR2_DIV_SHIFT; + sai_clk->div.width = CR2_DIV_WIDTH; + sai_clk->div.lock = &sai_clk->lock; + + /* set clock direction, we are the BCLK master */ + writel(CR2_BCD, base + I2S_CR2); + + hw = clk_hw_register_composite_pdata(dev, dev->of_node->name, + &pdata, 1, NULL, NULL, + &sai_clk->div.hw, + &clk_divider_ops, + &sai_clk->gate.hw, + &clk_gate_ops, + CLK_SET_RATE_GATE); + if (IS_ERR(hw)) + return PTR_ERR(hw); + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, hw); +} + +static const struct of_device_id of_fsl_sai_clk_ids[] = { + { .compatible = "fsl,vf610-sai-clock" }, + { } +}; +MODULE_DEVICE_TABLE(of, of_fsl_sai_clk_ids); + +static struct platform_driver fsl_sai_clk_driver = { + .probe = fsl_sai_clk_probe, + .driver = { + .name = "fsl-sai-clk", + .of_match_table = of_fsl_sai_clk_ids, + }, +}; +module_platform_driver(fsl_sai_clk_driver); + +MODULE_DESCRIPTION("Freescale SAI bitclock-as-a-clock driver"); +MODULE_AUTHOR("Michael Walle <michael@walle.cc>"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:fsl-sai-clk"); diff --git a/drivers/clk/clk-gate.c b/drivers/clk/clk-gate.c index 670053c58c1a..2ca1f2ac38a6 100644 --- a/drivers/clk/clk-gate.c +++ b/drivers/clk/clk-gate.c @@ -123,26 +123,18 @@ const struct clk_ops clk_gate_ops = { }; EXPORT_SYMBOL_GPL(clk_gate_ops); -/** - * clk_hw_register_gate - register a gate clock with the clock framework - * @dev: device that is registering this clock - * @name: name of this clock - * @parent_name: name of this clock's parent - * @flags: framework-specific flags for this clock - * @reg: register address to control gating of this clock - * @bit_idx: which bit in the register controls gating of this clock - * @clk_gate_flags: gate-specific flags for this clock - * @lock: shared register lock for this clock - */ -struct clk_hw *clk_hw_register_gate(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, +struct clk_hw *__clk_hw_register_gate(struct device *dev, + struct device_node *np, const char *name, + const char *parent_name, const struct clk_hw *parent_hw, + const struct clk_parent_data *parent_data, + unsigned long flags, void __iomem *reg, u8 bit_idx, u8 clk_gate_flags, spinlock_t *lock) { struct clk_gate *gate; struct clk_hw *hw; struct clk_init_data init = {}; - int ret; + int ret = -EINVAL; if (clk_gate_flags & CLK_GATE_HIWORD_MASK) { if (bit_idx > 15) { @@ -160,7 +152,12 @@ struct clk_hw *clk_hw_register_gate(struct device *dev, const char *name, init.ops = &clk_gate_ops; init.flags = flags; init.parent_names = parent_name ? &parent_name : NULL; - init.num_parents = parent_name ? 1 : 0; + init.parent_hws = parent_hw ? &parent_hw : NULL; + init.parent_data = parent_data; + if (parent_name || parent_hw || parent_data) + init.num_parents = 1; + else + init.num_parents = 0; /* struct clk_gate assignments */ gate->reg = reg; @@ -170,15 +167,19 @@ struct clk_hw *clk_hw_register_gate(struct device *dev, const char *name, gate->hw.init = &init; hw = &gate->hw; - ret = clk_hw_register(dev, hw); + if (dev || !np) + ret = clk_hw_register(dev, hw); + else if (np) + ret = of_clk_hw_register(np, hw); if (ret) { kfree(gate); hw = ERR_PTR(ret); } return hw; + } -EXPORT_SYMBOL_GPL(clk_hw_register_gate); +EXPORT_SYMBOL_GPL(__clk_hw_register_gate); struct clk *clk_register_gate(struct device *dev, const char *name, const char *parent_name, unsigned long flags, diff --git a/drivers/clk/clk-gpio.c b/drivers/clk/clk-gpio.c index 13304cf5f2a8..70397b4b5ffe 100644 --- a/drivers/clk/clk-gpio.c +++ b/drivers/clk/clk-gpio.c @@ -28,6 +28,26 @@ * parent - fixed parent. No clk_set_parent support */ +/** + * struct clk_gpio - gpio gated clock + * + * @hw: handle between common and hardware-specific interfaces + * @gpiod: gpio descriptor + * + * Clock with a gpio control for enabling and disabling the parent clock + * or switching between two parents by asserting or deasserting the gpio. + * + * Implements .enable, .disable and .is_enabled or + * .get_parent, .set_parent and .determine_rate depending on which clk_ops + * is used. + */ +struct clk_gpio { + struct clk_hw hw; + struct gpio_desc *gpiod; +}; + +#define to_clk_gpio(_hw) container_of(_hw, struct clk_gpio, hw) + static int clk_gpio_gate_enable(struct clk_hw *hw) { struct clk_gpio *clk = to_clk_gpio(hw); @@ -51,12 +71,11 @@ static int clk_gpio_gate_is_enabled(struct clk_hw *hw) return gpiod_get_value(clk->gpiod); } -const struct clk_ops clk_gpio_gate_ops = { +static const struct clk_ops clk_gpio_gate_ops = { .enable = clk_gpio_gate_enable, .disable = clk_gpio_gate_disable, .is_enabled = clk_gpio_gate_is_enabled, }; -EXPORT_SYMBOL_GPL(clk_gpio_gate_ops); static int clk_sleeping_gpio_gate_prepare(struct clk_hw *hw) { @@ -111,67 +130,49 @@ static int clk_gpio_mux_set_parent(struct clk_hw *hw, u8 index) return 0; } -const struct clk_ops clk_gpio_mux_ops = { +static const struct clk_ops clk_gpio_mux_ops = { .get_parent = clk_gpio_mux_get_parent, .set_parent = clk_gpio_mux_set_parent, .determine_rate = __clk_mux_determine_rate, }; -EXPORT_SYMBOL_GPL(clk_gpio_mux_ops); -static struct clk_hw *clk_register_gpio(struct device *dev, const char *name, - const char * const *parent_names, u8 num_parents, struct gpio_desc *gpiod, - unsigned long flags, const struct clk_ops *clk_gpio_ops) +static struct clk_hw *clk_register_gpio(struct device *dev, u8 num_parents, + struct gpio_desc *gpiod, + const struct clk_ops *clk_gpio_ops) { struct clk_gpio *clk_gpio; struct clk_hw *hw; struct clk_init_data init = {}; int err; + const struct clk_parent_data gpio_parent_data[] = { + { .index = 0 }, + { .index = 1 }, + }; - if (dev) - clk_gpio = devm_kzalloc(dev, sizeof(*clk_gpio), GFP_KERNEL); - else - clk_gpio = kzalloc(sizeof(*clk_gpio), GFP_KERNEL); - + clk_gpio = devm_kzalloc(dev, sizeof(*clk_gpio), GFP_KERNEL); if (!clk_gpio) return ERR_PTR(-ENOMEM); - init.name = name; + init.name = dev->of_node->name; init.ops = clk_gpio_ops; - init.flags = flags; - init.parent_names = parent_names; + init.parent_data = gpio_parent_data; init.num_parents = num_parents; + init.flags = CLK_SET_RATE_PARENT; clk_gpio->gpiod = gpiod; clk_gpio->hw.init = &init; hw = &clk_gpio->hw; - if (dev) - err = devm_clk_hw_register(dev, hw); - else - err = clk_hw_register(NULL, hw); - - if (!err) - return hw; - - if (!dev) { - kfree(clk_gpio); - } + err = devm_clk_hw_register(dev, hw); + if (err) + return ERR_PTR(err); - return ERR_PTR(err); + return hw; } -/** - * clk_hw_register_gpio_gate - register a gpio clock gate with the clock - * framework - * @dev: device that is registering this clock - * @name: name of this clock - * @parent_name: name of this clock's parent - * @gpiod: gpio descriptor to gate this clock - * @flags: clock flags - */ -struct clk_hw *clk_hw_register_gpio_gate(struct device *dev, const char *name, - const char *parent_name, struct gpio_desc *gpiod, - unsigned long flags) +static struct clk_hw *clk_hw_register_gpio_gate(struct device *dev, + int num_parents, + struct gpio_desc *gpiod) { const struct clk_ops *ops; @@ -180,88 +181,36 @@ struct clk_hw *clk_hw_register_gpio_gate(struct device *dev, const char *name, else ops = &clk_gpio_gate_ops; - return clk_register_gpio(dev, name, - (parent_name ? &parent_name : NULL), - (parent_name ? 1 : 0), gpiod, flags, ops); + return clk_register_gpio(dev, num_parents, gpiod, ops); } -EXPORT_SYMBOL_GPL(clk_hw_register_gpio_gate); -struct clk *clk_register_gpio_gate(struct device *dev, const char *name, - const char *parent_name, struct gpio_desc *gpiod, - unsigned long flags) +static struct clk_hw *clk_hw_register_gpio_mux(struct device *dev, + struct gpio_desc *gpiod) { - struct clk_hw *hw; - - hw = clk_hw_register_gpio_gate(dev, name, parent_name, gpiod, flags); - if (IS_ERR(hw)) - return ERR_CAST(hw); - return hw->clk; + return clk_register_gpio(dev, 2, gpiod, &clk_gpio_mux_ops); } -EXPORT_SYMBOL_GPL(clk_register_gpio_gate); - -/** - * clk_hw_register_gpio_mux - register a gpio clock mux with the clock framework - * @dev: device that is registering this clock - * @name: name of this clock - * @parent_names: names of this clock's parents - * @num_parents: number of parents listed in @parent_names - * @gpiod: gpio descriptor to gate this clock - * @flags: clock flags - */ -struct clk_hw *clk_hw_register_gpio_mux(struct device *dev, const char *name, - const char * const *parent_names, u8 num_parents, struct gpio_desc *gpiod, - unsigned long flags) -{ - if (num_parents != 2) { - pr_err("mux-clock %s must have 2 parents\n", name); - return ERR_PTR(-EINVAL); - } - - return clk_register_gpio(dev, name, parent_names, num_parents, - gpiod, flags, &clk_gpio_mux_ops); -} -EXPORT_SYMBOL_GPL(clk_hw_register_gpio_mux); - -struct clk *clk_register_gpio_mux(struct device *dev, const char *name, - const char * const *parent_names, u8 num_parents, struct gpio_desc *gpiod, - unsigned long flags) -{ - struct clk_hw *hw; - - hw = clk_hw_register_gpio_mux(dev, name, parent_names, num_parents, - gpiod, flags); - if (IS_ERR(hw)) - return ERR_CAST(hw); - return hw->clk; -} -EXPORT_SYMBOL_GPL(clk_register_gpio_mux); static int gpio_clk_driver_probe(struct platform_device *pdev) { - struct device_node *node = pdev->dev.of_node; - const char **parent_names, *gpio_name; + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + const char *gpio_name; unsigned int num_parents; struct gpio_desc *gpiod; - struct clk *clk; + struct clk_hw *hw; bool is_mux; int ret; + is_mux = of_device_is_compatible(node, "gpio-mux-clock"); + num_parents = of_clk_get_parent_count(node); - if (num_parents) { - parent_names = devm_kcalloc(&pdev->dev, num_parents, - sizeof(char *), GFP_KERNEL); - if (!parent_names) - return -ENOMEM; - - of_clk_parent_fill(node, parent_names, num_parents); - } else { - parent_names = NULL; + if (is_mux && num_parents != 2) { + dev_err(dev, "mux-clock must have 2 parents\n"); + return -EINVAL; } - is_mux = of_device_is_compatible(node, "gpio-mux-clock"); - gpio_name = is_mux ? "select" : "enable"; - gpiod = devm_gpiod_get(&pdev->dev, gpio_name, GPIOD_OUT_LOW); + gpiod = devm_gpiod_get(dev, gpio_name, GPIOD_OUT_LOW); if (IS_ERR(gpiod)) { ret = PTR_ERR(gpiod); if (ret == -EPROBE_DEFER) @@ -275,16 +224,13 @@ static int gpio_clk_driver_probe(struct platform_device *pdev) } if (is_mux) - clk = clk_register_gpio_mux(&pdev->dev, node->name, - parent_names, num_parents, gpiod, 0); + hw = clk_hw_register_gpio_mux(dev, gpiod); else - clk = clk_register_gpio_gate(&pdev->dev, node->name, - parent_names ? parent_names[0] : NULL, gpiod, - CLK_SET_RATE_PARENT); - if (IS_ERR(clk)) - return PTR_ERR(clk); + hw = clk_hw_register_gpio_gate(dev, num_parents, gpiod); + if (IS_ERR(hw)) + return PTR_ERR(hw); - return of_clk_add_provider(node, of_clk_src_simple_get, clk); + return devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, hw); } static const struct of_device_id gpio_clk_match_table[] = { diff --git a/drivers/clk/clk-mux.c b/drivers/clk/clk-mux.c index 570b6e5b603b..e54e79714818 100644 --- a/drivers/clk/clk-mux.c +++ b/drivers/clk/clk-mux.c @@ -145,17 +145,19 @@ const struct clk_ops clk_mux_ro_ops = { }; EXPORT_SYMBOL_GPL(clk_mux_ro_ops); -struct clk_hw *clk_hw_register_mux_table(struct device *dev, const char *name, - const char * const *parent_names, u8 num_parents, - unsigned long flags, - void __iomem *reg, u8 shift, u32 mask, +struct clk_hw *__clk_hw_register_mux(struct device *dev, struct device_node *np, + const char *name, u8 num_parents, + const char * const *parent_names, + const struct clk_hw **parent_hws, + const struct clk_parent_data *parent_data, + unsigned long flags, void __iomem *reg, u8 shift, u32 mask, u8 clk_mux_flags, u32 *table, spinlock_t *lock) { struct clk_mux *mux; struct clk_hw *hw; struct clk_init_data init = {}; u8 width = 0; - int ret; + int ret = -EINVAL; if (clk_mux_flags & CLK_MUX_HIWORD_MASK) { width = fls(mask) - ffs(mask) + 1; @@ -177,6 +179,8 @@ struct clk_hw *clk_hw_register_mux_table(struct device *dev, const char *name, init.ops = &clk_mux_ops; init.flags = flags; init.parent_names = parent_names; + init.parent_data = parent_data; + init.parent_hws = parent_hws; init.num_parents = num_parents; /* struct clk_mux assignments */ @@ -189,7 +193,10 @@ struct clk_hw *clk_hw_register_mux_table(struct device *dev, const char *name, mux->hw.init = &init; hw = &mux->hw; - ret = clk_hw_register(dev, hw); + if (dev || !np) + ret = clk_hw_register(dev, hw); + else if (np) + ret = of_clk_hw_register(np, hw); if (ret) { kfree(mux); hw = ERR_PTR(ret); @@ -197,53 +204,24 @@ struct clk_hw *clk_hw_register_mux_table(struct device *dev, const char *name, return hw; } -EXPORT_SYMBOL_GPL(clk_hw_register_mux_table); +EXPORT_SYMBOL_GPL(__clk_hw_register_mux); struct clk *clk_register_mux_table(struct device *dev, const char *name, const char * const *parent_names, u8 num_parents, - unsigned long flags, - void __iomem *reg, u8 shift, u32 mask, + unsigned long flags, void __iomem *reg, u8 shift, u32 mask, u8 clk_mux_flags, u32 *table, spinlock_t *lock) { struct clk_hw *hw; - hw = clk_hw_register_mux_table(dev, name, parent_names, num_parents, - flags, reg, shift, mask, clk_mux_flags, - table, lock); + hw = clk_hw_register_mux_table(dev, name, parent_names, + num_parents, flags, reg, shift, mask, + clk_mux_flags, table, lock); if (IS_ERR(hw)) return ERR_CAST(hw); return hw->clk; } EXPORT_SYMBOL_GPL(clk_register_mux_table); -struct clk *clk_register_mux(struct device *dev, const char *name, - const char * const *parent_names, u8 num_parents, - unsigned long flags, - void __iomem *reg, u8 shift, u8 width, - u8 clk_mux_flags, spinlock_t *lock) -{ - u32 mask = BIT(width) - 1; - - return clk_register_mux_table(dev, name, parent_names, num_parents, - flags, reg, shift, mask, clk_mux_flags, - NULL, lock); -} -EXPORT_SYMBOL_GPL(clk_register_mux); - -struct clk_hw *clk_hw_register_mux(struct device *dev, const char *name, - const char * const *parent_names, u8 num_parents, - unsigned long flags, - void __iomem *reg, u8 shift, u8 width, - u8 clk_mux_flags, spinlock_t *lock) -{ - u32 mask = BIT(width) - 1; - - return clk_hw_register_mux_table(dev, name, parent_names, num_parents, - flags, reg, shift, mask, clk_mux_flags, - NULL, lock); -} -EXPORT_SYMBOL_GPL(clk_hw_register_mux); - void clk_unregister_mux(struct clk *clk) { struct clk_mux *mux; diff --git a/drivers/clk/clk-plldig.c b/drivers/clk/clk-plldig.c new file mode 100644 index 000000000000..25020164b89e --- /dev/null +++ b/drivers/clk/clk-plldig.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 NXP + * + * Clock driver for LS1028A Display output interfaces(LCD, DPHY). + */ + +#include <linux/clk-provider.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/bitfield.h> + +/* PLLDIG register offsets and bit masks */ +#define PLLDIG_REG_PLLSR 0x24 +#define PLLDIG_LOCK_MASK BIT(2) +#define PLLDIG_REG_PLLDV 0x28 +#define PLLDIG_MFD_MASK GENMASK(7, 0) +#define PLLDIG_RFDPHI1_MASK GENMASK(30, 25) +#define PLLDIG_REG_PLLFM 0x2c +#define PLLDIG_SSCGBYP_ENABLE BIT(30) +#define PLLDIG_REG_PLLFD 0x30 +#define PLLDIG_FDEN BIT(30) +#define PLLDIG_FRAC_MASK GENMASK(15, 0) +#define PLLDIG_REG_PLLCAL1 0x38 +#define PLLDIG_REG_PLLCAL2 0x3c + +/* Range of the VCO frequencies, in Hz */ +#define PLLDIG_MIN_VCO_FREQ 650000000 +#define PLLDIG_MAX_VCO_FREQ 1300000000 + +/* Range of the output frequencies, in Hz */ +#define PHI1_MIN_FREQ 27000000UL +#define PHI1_MAX_FREQ 600000000UL + +/* Maximum value of the reduced frequency divider */ +#define MAX_RFDPHI1 63UL + +/* Best value of multiplication factor divider */ +#define PLLDIG_DEFAULT_MFD 44 + +/* + * Denominator part of the fractional part of the + * loop multiplication factor. + */ +#define MFDEN 20480 + +static const struct clk_parent_data parent_data[] = { + { .index = 0 }, +}; + +struct clk_plldig { + struct clk_hw hw; + void __iomem *regs; + unsigned int vco_freq; +}; + +#define to_clk_plldig(_hw) container_of(_hw, struct clk_plldig, hw) + +static int plldig_enable(struct clk_hw *hw) +{ + struct clk_plldig *data = to_clk_plldig(hw); + u32 val; + + val = readl(data->regs + PLLDIG_REG_PLLFM); + /* + * Use Bypass mode with PLL off by default, the frequency overshoot + * detector output was disable. SSCG Bypass mode should be enable. + */ + val |= PLLDIG_SSCGBYP_ENABLE; + writel(val, data->regs + PLLDIG_REG_PLLFM); + + return 0; +} + +static void plldig_disable(struct clk_hw *hw) +{ + struct clk_plldig *data = to_clk_plldig(hw); + u32 val; + + val = readl(data->regs + PLLDIG_REG_PLLFM); + + val &= ~PLLDIG_SSCGBYP_ENABLE; + val |= FIELD_PREP(PLLDIG_SSCGBYP_ENABLE, 0x0); + + writel(val, data->regs + PLLDIG_REG_PLLFM); +} + +static int plldig_is_enabled(struct clk_hw *hw) +{ + struct clk_plldig *data = to_clk_plldig(hw); + + return readl(data->regs + PLLDIG_REG_PLLFM) & + PLLDIG_SSCGBYP_ENABLE; +} + +static unsigned long plldig_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_plldig *data = to_clk_plldig(hw); + u32 val, rfdphi1; + + val = readl(data->regs + PLLDIG_REG_PLLDV); + + /* Check if PLL is bypassed */ + if (val & PLLDIG_SSCGBYP_ENABLE) + return parent_rate; + + rfdphi1 = FIELD_GET(PLLDIG_RFDPHI1_MASK, val); + + /* + * If RFDPHI1 has a value of 1 the VCO frequency is also divided by + * one. + */ + if (!rfdphi1) + rfdphi1 = 1; + + return DIV_ROUND_UP(data->vco_freq, rfdphi1); +} + +static unsigned long plldig_calc_target_div(unsigned long vco_freq, + unsigned long target_rate) +{ + unsigned long div; + + div = DIV_ROUND_CLOSEST(vco_freq, target_rate); + div = clamp(div, 1UL, MAX_RFDPHI1); + + return div; +} + +static int plldig_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_plldig *data = to_clk_plldig(hw); + unsigned int div; + + req->rate = clamp(req->rate, PHI1_MIN_FREQ, PHI1_MAX_FREQ); + div = plldig_calc_target_div(data->vco_freq, req->rate); + req->rate = DIV_ROUND_UP(data->vco_freq, div); + + return 0; +} + +static int plldig_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_plldig *data = to_clk_plldig(hw); + unsigned int val, cond; + unsigned int rfdphi1; + + rate = clamp(rate, PHI1_MIN_FREQ, PHI1_MAX_FREQ); + rfdphi1 = plldig_calc_target_div(data->vco_freq, rate); + + /* update the divider value */ + val = readl(data->regs + PLLDIG_REG_PLLDV); + val &= ~PLLDIG_RFDPHI1_MASK; + val |= FIELD_PREP(PLLDIG_RFDPHI1_MASK, rfdphi1); + writel(val, data->regs + PLLDIG_REG_PLLDV); + + /* waiting for old lock state to clear */ + udelay(200); + + /* Wait until PLL is locked or timeout */ + return readl_poll_timeout_atomic(data->regs + PLLDIG_REG_PLLSR, cond, + cond & PLLDIG_LOCK_MASK, 0, + USEC_PER_MSEC); +} + +static const struct clk_ops plldig_clk_ops = { + .enable = plldig_enable, + .disable = plldig_disable, + .is_enabled = plldig_is_enabled, + .recalc_rate = plldig_recalc_rate, + .determine_rate = plldig_determine_rate, + .set_rate = plldig_set_rate, +}; + +static int plldig_init(struct clk_hw *hw) +{ + struct clk_plldig *data = to_clk_plldig(hw); + struct clk_hw *parent = clk_hw_get_parent(hw); + unsigned long parent_rate; + unsigned long val; + unsigned long long lltmp; + unsigned int mfd, fracdiv = 0; + + if (!parent) + return -EINVAL; + + parent_rate = clk_hw_get_rate(parent); + + if (data->vco_freq) { + mfd = data->vco_freq / parent_rate; + lltmp = data->vco_freq % parent_rate; + lltmp *= MFDEN; + do_div(lltmp, parent_rate); + fracdiv = lltmp; + } else { + mfd = PLLDIG_DEFAULT_MFD; + data->vco_freq = parent_rate * mfd; + } + + val = FIELD_PREP(PLLDIG_MFD_MASK, mfd); + writel(val, data->regs + PLLDIG_REG_PLLDV); + + /* Enable fractional divider */ + if (fracdiv) { + val = FIELD_PREP(PLLDIG_FRAC_MASK, fracdiv); + val |= PLLDIG_FDEN; + writel(val, data->regs + PLLDIG_REG_PLLFD); + } + + return 0; +} + +static int plldig_clk_probe(struct platform_device *pdev) +{ + struct clk_plldig *data; + struct device *dev = &pdev->dev; + int ret; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(data->regs)) + return PTR_ERR(data->regs); + + data->hw.init = CLK_HW_INIT_PARENTS_DATA("dpclk", + parent_data, + &plldig_clk_ops, + 0); + + ret = devm_clk_hw_register(dev, &data->hw); + if (ret) { + dev_err(dev, "failed to register %s clock\n", + dev->of_node->name); + return ret; + } + + ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &data->hw); + if (ret) { + dev_err(dev, "unable to add clk provider\n"); + return ret; + } + + /* + * The frequency of the VCO cannot be changed during runtime. + * Therefore, let the user specify a desired frequency. + */ + if (!of_property_read_u32(dev->of_node, "fsl,vco-hz", + &data->vco_freq)) { + if (data->vco_freq < PLLDIG_MIN_VCO_FREQ || + data->vco_freq > PLLDIG_MAX_VCO_FREQ) + return -EINVAL; + } + + return plldig_init(&data->hw); +} + +static const struct of_device_id plldig_clk_id[] = { + { .compatible = "fsl,ls1028a-plldig" }, + { } +}; +MODULE_DEVICE_TABLE(of, plldig_clk_id); + +static struct platform_driver plldig_clk_driver = { + .driver = { + .name = "plldig-clock", + .of_match_table = plldig_clk_id, + }, + .probe = plldig_clk_probe, +}; +module_platform_driver(plldig_clk_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Wen He <wen.he_1@nxp.com>"); +MODULE_DESCRIPTION("LS1028A Display output interface pixel clock driver"); diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index bed140f7375f..d5946f7486d6 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -342,6 +342,32 @@ static const struct clockgen_muxinfo ls1046a_hwa2 = { }, }; +static const struct clockgen_muxinfo ls1088a_hwa1 = { + { + {}, + { CLKSEL_VALID, CGA_PLL1, PLL_DIV1 }, + { CLKSEL_VALID, CGA_PLL1, PLL_DIV2 }, + { CLKSEL_VALID, CGA_PLL1, PLL_DIV3 }, + { CLKSEL_VALID, CGA_PLL1, PLL_DIV4 }, + {}, + { CLKSEL_VALID, CGA_PLL2, PLL_DIV2 }, + { CLKSEL_VALID, CGA_PLL2, PLL_DIV3 }, + }, +}; + +static const struct clockgen_muxinfo ls1088a_hwa2 = { + { + {}, + { CLKSEL_VALID, CGA_PLL2, PLL_DIV1 }, + { CLKSEL_VALID, CGA_PLL2, PLL_DIV2 }, + { CLKSEL_VALID, CGA_PLL2, PLL_DIV3 }, + { CLKSEL_VALID, CGA_PLL2, PLL_DIV4 }, + {}, + { CLKSEL_VALID, CGA_PLL1, PLL_DIV2 }, + { CLKSEL_VALID, CGA_PLL1, PLL_DIV3 }, + }, +}; + static const struct clockgen_muxinfo ls1012a_cmux = { { [0] = { CLKSEL_VALID, CGA_PLL1, PLL_DIV1 }, @@ -607,6 +633,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_groups = { &clockgen2_cmux_cga12 }, + .hwaccel = { + &ls1088a_hwa1, &ls1088a_hwa2 + }, .cmux_to_group = { 0, 0, -1 }, diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 772258de2d1f..f0f2b599fd7e 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -429,7 +429,7 @@ static void clk_core_fill_parent_index(struct clk_core *core, u8 index) parent = ERR_PTR(-EPROBE_DEFER); } else { parent = clk_core_get(core, index); - if (IS_ERR(parent) && PTR_ERR(parent) == -ENOENT && entry->name) + if (PTR_ERR(parent) == -ENOENT && entry->name) parent = clk_core_lookup(entry->name); } @@ -2996,6 +2996,41 @@ static int clk_dump_show(struct seq_file *s, void *data) } DEFINE_SHOW_ATTRIBUTE(clk_dump); +#undef CLOCK_ALLOW_WRITE_DEBUGFS +#ifdef CLOCK_ALLOW_WRITE_DEBUGFS +/* + * This can be dangerous, therefore don't provide any real compile time + * configuration option for this feature. + * People who want to use this will need to modify the source code directly. + */ +static int clk_rate_set(void *data, u64 val) +{ + struct clk_core *core = data; + int ret; + + clk_prepare_lock(); + ret = clk_core_set_rate_nolock(core, val); + clk_prepare_unlock(); + + return ret; +} + +#define clk_rate_mode 0644 +#else +#define clk_rate_set NULL +#define clk_rate_mode 0444 +#endif + +static int clk_rate_get(void *data, u64 *val) +{ + struct clk_core *core = data; + + *val = core->rate; + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(clk_rate_fops, clk_rate_get, clk_rate_set, "%llu\n"); + static const struct { unsigned long flag; const char *name; @@ -3145,7 +3180,8 @@ static void clk_debug_create_one(struct clk_core *core, struct dentry *pdentry) root = debugfs_create_dir(core->name, pdentry); core->dentry = root; - debugfs_create_ulong("clk_rate", 0444, root, &core->rate); + debugfs_create_file("clk_rate", clk_rate_mode, root, core, + &clk_rate_fops); debugfs_create_file("clk_min_rate", 0444, root, core, &clk_min_rate_fops); debugfs_create_file("clk_max_rate", 0444, root, core, &clk_max_rate_fops); debugfs_create_ulong("clk_accuracy", 0444, root, &core->accuracy); @@ -3338,6 +3374,26 @@ static int __clk_core_init(struct clk_core *core) goto out; } + /* + * optional platform-specific magic + * + * The .init callback is not used by any of the basic clock types, but + * exists for weird hardware that must perform initialization magic for + * CCF to get an accurate view of clock for any other callbacks. It may + * also be used needs to perform dynamic allocations. Such allocation + * must be freed in the terminate() callback. + * This callback shall not be used to initialize the parameters state, + * such as rate, parent, etc ... + * + * If it exist, this callback should called before any other callback of + * the clock + */ + if (core->ops->init) { + ret = core->ops->init(core->hw); + if (ret) + goto out; + } + core->parent = __clk_init_parent(core); /* @@ -3363,17 +3419,6 @@ static int __clk_core_init(struct clk_core *core) } /* - * optional platform-specific magic - * - * The .init callback is not used by any of the basic clock types, but - * exists for weird hardware that must perform initialization magic. - * Please consider other ways of solving initialization problems before - * using this callback, as its use is discouraged. - */ - if (core->ops->init) - core->ops->init(core->hw); - - /* * Set clk's accuracy. The preferred method is to use * .recalc_accuracy. For simple clocks and lazy developers the default * fallback is to use the parent's accuracy. If a clock doesn't have a @@ -3427,13 +3472,18 @@ static int __clk_core_init(struct clk_core *core) unsigned long flags; ret = clk_core_prepare(core); - if (ret) + if (ret) { + pr_warn("%s: critical clk '%s' failed to prepare\n", + __func__, core->name); goto out; + } flags = clk_enable_lock(); ret = clk_core_enable(core); clk_enable_unlock(flags); if (ret) { + pr_warn("%s: critical clk '%s' failed to enable\n", + __func__, core->name); clk_core_unprepare(core); goto out; } @@ -3733,6 +3783,28 @@ fail_out: } /** + * dev_or_parent_of_node() - Get device node of @dev or @dev's parent + * @dev: Device to get device node of + * + * Return: device node pointer of @dev, or the device node pointer of + * @dev->parent if dev doesn't have a device node, or NULL if neither + * @dev or @dev->parent have a device node. + */ +static struct device_node *dev_or_parent_of_node(struct device *dev) +{ + struct device_node *np; + + if (!dev) + return NULL; + + np = dev_of_node(dev); + if (!np) + np = dev_of_node(dev->parent); + + return np; +} + +/** * clk_register - allocate a new clock, register it and return an opaque cookie * @dev: device that is registering this clock * @hw: link to hardware-specific clock data @@ -3747,7 +3819,7 @@ fail_out: */ struct clk *clk_register(struct device *dev, struct clk_hw *hw) { - return __clk_register(dev, dev_of_node(dev), hw); + return __clk_register(dev, dev_or_parent_of_node(dev), hw); } EXPORT_SYMBOL_GPL(clk_register); @@ -3763,7 +3835,8 @@ EXPORT_SYMBOL_GPL(clk_register); */ int clk_hw_register(struct device *dev, struct clk_hw *hw) { - return PTR_ERR_OR_ZERO(__clk_register(dev, dev_of_node(dev), hw)); + return PTR_ERR_OR_ZERO(__clk_register(dev, dev_or_parent_of_node(dev), + hw)); } EXPORT_SYMBOL_GPL(clk_hw_register); @@ -3866,6 +3939,7 @@ static void clk_core_evict_parent_cache(struct clk_core *core) void clk_unregister(struct clk *clk) { unsigned long flags; + const struct clk_ops *ops; if (!clk || WARN_ON_ONCE(IS_ERR(clk))) return; @@ -3874,7 +3948,8 @@ void clk_unregister(struct clk *clk) clk_prepare_lock(); - if (clk->core->ops == &clk_nodrv_ops) { + ops = clk->core->ops; + if (ops == &clk_nodrv_ops) { pr_err("%s: unregistered clock: %s\n", __func__, clk->core->name); goto unlock; @@ -3887,6 +3962,9 @@ void clk_unregister(struct clk *clk) clk->core->ops = &clk_nodrv_ops; clk_enable_unlock(flags); + if (ops->terminate) + ops->terminate(clk->core->hw); + if (!hlist_empty(&clk->core->children)) { struct clk_core *child; struct hlist_node *t; diff --git a/drivers/clk/imx/Kconfig b/drivers/clk/imx/Kconfig index 1ac0c7990392..01eadee88d66 100644 --- a/drivers/clk/imx/Kconfig +++ b/drivers/clk/imx/Kconfig @@ -20,6 +20,12 @@ config CLK_IMX8MN help Build the driver for i.MX8MN CCM Clock Driver +config CLK_IMX8MP + bool "IMX8MP CCM Clock Driver" + depends on ARCH_MXC && ARM64 + help + Build the driver for i.MX8MP CCM Clock Driver + config CLK_IMX8MQ bool "IMX8MQ CCM Clock Driver" depends on ARCH_MXC && ARM64 diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile index 77a3d714f1d5..928f874c73d2 100644 --- a/drivers/clk/imx/Makefile +++ b/drivers/clk/imx/Makefile @@ -18,7 +18,7 @@ obj-$(CONFIG_MXC_CLK) += \ clk-pllv2.o \ clk-pllv3.o \ clk-pllv4.o \ - clk-sccg-pll.o \ + clk-sscg-pll.o \ clk-pll14xx.o obj-$(CONFIG_MXC_CLK_SCU) += \ @@ -27,6 +27,7 @@ obj-$(CONFIG_MXC_CLK_SCU) += \ obj-$(CONFIG_CLK_IMX8MM) += clk-imx8mm.o obj-$(CONFIG_CLK_IMX8MN) += clk-imx8mn.o +obj-$(CONFIG_CLK_IMX8MP) += clk-imx8mp.o obj-$(CONFIG_CLK_IMX8MQ) += clk-imx8mq.o obj-$(CONFIG_CLK_IMX8QXP) += clk-imx8qxp.o clk-imx8qxp-lpcg.o diff --git a/drivers/clk/imx/clk-composite-7ulp.c b/drivers/clk/imx/clk-composite-7ulp.c index 060f8600ea0d..b9efcc8a855d 100644 --- a/drivers/clk/imx/clk-composite-7ulp.c +++ b/drivers/clk/imx/clk-composite-7ulp.c @@ -21,7 +21,7 @@ #define PCG_PCD_WIDTH 3 #define PCG_PCD_MASK 0x7 -struct clk_hw *imx7ulp_clk_composite(const char *name, +struct clk_hw *imx7ulp_clk_hw_composite(const char *name, const char * const *parent_names, int num_parents, bool mux_present, bool rate_present, bool gate_present, diff --git a/drivers/clk/imx/clk-composite-8m.c b/drivers/clk/imx/clk-composite-8m.c index d3486ee79ab5..20f7c91c03d2 100644 --- a/drivers/clk/imx/clk-composite-8m.c +++ b/drivers/clk/imx/clk-composite-8m.c @@ -123,7 +123,7 @@ static const struct clk_ops imx8m_clk_composite_divider_ops = { .set_rate = imx8m_clk_composite_divider_set_rate, }; -struct clk *imx8m_clk_composite_flags(const char *name, +struct clk_hw *imx8m_clk_hw_composite_flags(const char *name, const char * const *parent_names, int num_parents, void __iomem *reg, unsigned long flags) @@ -171,7 +171,7 @@ struct clk *imx8m_clk_composite_flags(const char *name, if (IS_ERR(hw)) goto fail; - return hw->clk; + return hw; fail: kfree(gate); diff --git a/drivers/clk/imx/clk-divider-gate.c b/drivers/clk/imx/clk-divider-gate.c index 2a8352a316c7..0322a843d245 100644 --- a/drivers/clk/imx/clk-divider-gate.c +++ b/drivers/clk/imx/clk-divider-gate.c @@ -43,7 +43,7 @@ static unsigned long clk_divider_gate_recalc_rate(struct clk_hw *hw, { struct clk_divider_gate *div_gate = to_clk_divider_gate(hw); struct clk_divider *div = to_clk_divider(hw); - unsigned long flags = 0; + unsigned long flags; unsigned int val; spin_lock_irqsave(div->lock, flags); @@ -75,7 +75,7 @@ static int clk_divider_gate_set_rate(struct clk_hw *hw, unsigned long rate, { struct clk_divider_gate *div_gate = to_clk_divider_gate(hw); struct clk_divider *div = to_clk_divider(hw); - unsigned long flags = 0; + unsigned long flags; int value; u32 val; @@ -104,7 +104,7 @@ static int clk_divider_enable(struct clk_hw *hw) { struct clk_divider_gate *div_gate = to_clk_divider_gate(hw); struct clk_divider *div = to_clk_divider(hw); - unsigned long flags = 0; + unsigned long flags; u32 val; if (!div_gate->cached_val) { @@ -127,7 +127,7 @@ static void clk_divider_disable(struct clk_hw *hw) { struct clk_divider_gate *div_gate = to_clk_divider_gate(hw); struct clk_divider *div = to_clk_divider(hw); - unsigned long flags = 0; + unsigned long flags; u32 val; spin_lock_irqsave(div->lock, flags); @@ -167,13 +167,13 @@ static const struct clk_ops clk_divider_gate_ops = { }; /* - * NOTE: In order to resue the most code from the common divider, + * NOTE: In order to reuse the most code from the common divider, * we also design our divider following the way that provids an extra * clk_divider_flags, however it's fixed to CLK_DIVIDER_ONE_BASED by * default as our HW is. Besides that it supports only CLK_DIVIDER_READ_ONLY * flag which can be specified by user flexibly. */ -struct clk_hw *imx_clk_divider_gate(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_divider_gate(const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 shift, u8 width, u8 clk_divider_flags, const struct clk_div_table *table, diff --git a/drivers/clk/imx/clk-frac-pll.c b/drivers/clk/imx/clk-frac-pll.c index fece503e3610..101e0a300376 100644 --- a/drivers/clk/imx/clk-frac-pll.c +++ b/drivers/clk/imx/clk-frac-pll.c @@ -201,8 +201,9 @@ static const struct clk_ops clk_frac_pll_ops = { .set_rate = clk_pll_set_rate, }; -struct clk *imx_clk_frac_pll(const char *name, const char *parent_name, - void __iomem *base) +struct clk_hw *imx_clk_hw_frac_pll(const char *name, + const char *parent_name, + void __iomem *base) { struct clk_init_data init; struct clk_frac_pll *pll; @@ -230,5 +231,5 @@ struct clk *imx_clk_frac_pll(const char *name, const char *parent_name, return ERR_PTR(ret); } - return hw->clk; + return hw; } diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index 60f2de851f39..ba33c79158de 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -598,7 +598,10 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) } hws[IMX6QDL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); - hws[IMX6QDL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock); + if (clk_on_imx6q() || clk_on_imx6qp()) + hws[IMX6QDL_CLK_PLL4_AUDIO_DIV] = imx_clk_hw_fixed_factor("pll4_audio_div", "pll4_post_div", 1, 1); + else + hws[IMX6QDL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock); hws[IMX6QDL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); hws[IMX6QDL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); diff --git a/drivers/clk/imx/clk-imx7ulp.c b/drivers/clk/imx/clk-imx7ulp.c index 281191b55b3a..0620d6c8c072 100644 --- a/drivers/clk/imx/clk-imx7ulp.c +++ b/drivers/clk/imx/clk-imx7ulp.c @@ -59,7 +59,7 @@ static struct clk **pcc3_uart_clks[ARRAY_SIZE(pcc3_uart_clk_ids) + 1] __initdata static void __init imx7ulp_clk_scg1_init(struct device_node *np) { struct clk_hw_onecell_data *clk_data; - struct clk_hw **clks; + struct clk_hw **hws; void __iomem *base; clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_SCG1_END), @@ -68,76 +68,76 @@ static void __init imx7ulp_clk_scg1_init(struct device_node *np) return; clk_data->num = IMX7ULP_CLK_SCG1_END; - clks = clk_data->hws; + hws = clk_data->hws; - clks[IMX7ULP_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); + hws[IMX7ULP_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); - clks[IMX7ULP_CLK_ROSC] = imx_obtain_fixed_clk_hw(np, "rosc"); - clks[IMX7ULP_CLK_SOSC] = imx_obtain_fixed_clk_hw(np, "sosc"); - clks[IMX7ULP_CLK_SIRC] = imx_obtain_fixed_clk_hw(np, "sirc"); - clks[IMX7ULP_CLK_FIRC] = imx_obtain_fixed_clk_hw(np, "firc"); - clks[IMX7ULP_CLK_UPLL] = imx_obtain_fixed_clk_hw(np, "upll"); + hws[IMX7ULP_CLK_ROSC] = imx_obtain_fixed_clk_hw(np, "rosc"); + hws[IMX7ULP_CLK_SOSC] = imx_obtain_fixed_clk_hw(np, "sosc"); + hws[IMX7ULP_CLK_SIRC] = imx_obtain_fixed_clk_hw(np, "sirc"); + hws[IMX7ULP_CLK_FIRC] = imx_obtain_fixed_clk_hw(np, "firc"); + hws[IMX7ULP_CLK_UPLL] = imx_obtain_fixed_clk_hw(np, "upll"); /* SCG1 */ base = of_iomap(np, 0); WARN_ON(!base); /* NOTE: xPLL config can't be changed when xPLL is enabled */ - clks[IMX7ULP_CLK_APLL_PRE_SEL] = imx_clk_hw_mux_flags("apll_pre_sel", base + 0x508, 0, 1, pll_pre_sels, ARRAY_SIZE(pll_pre_sels), CLK_SET_PARENT_GATE); - clks[IMX7ULP_CLK_SPLL_PRE_SEL] = imx_clk_hw_mux_flags("spll_pre_sel", base + 0x608, 0, 1, pll_pre_sels, ARRAY_SIZE(pll_pre_sels), CLK_SET_PARENT_GATE); + hws[IMX7ULP_CLK_APLL_PRE_SEL] = imx_clk_hw_mux_flags("apll_pre_sel", base + 0x508, 0, 1, pll_pre_sels, ARRAY_SIZE(pll_pre_sels), CLK_SET_PARENT_GATE); + hws[IMX7ULP_CLK_SPLL_PRE_SEL] = imx_clk_hw_mux_flags("spll_pre_sel", base + 0x608, 0, 1, pll_pre_sels, ARRAY_SIZE(pll_pre_sels), CLK_SET_PARENT_GATE); /* name parent_name reg shift width flags */ - clks[IMX7ULP_CLK_APLL_PRE_DIV] = imx_clk_hw_divider_flags("apll_pre_div", "apll_pre_sel", base + 0x508, 8, 3, CLK_SET_RATE_GATE); - clks[IMX7ULP_CLK_SPLL_PRE_DIV] = imx_clk_hw_divider_flags("spll_pre_div", "spll_pre_sel", base + 0x608, 8, 3, CLK_SET_RATE_GATE); + hws[IMX7ULP_CLK_APLL_PRE_DIV] = imx_clk_hw_divider_flags("apll_pre_div", "apll_pre_sel", base + 0x508, 8, 3, CLK_SET_RATE_GATE); + hws[IMX7ULP_CLK_SPLL_PRE_DIV] = imx_clk_hw_divider_flags("spll_pre_div", "spll_pre_sel", base + 0x608, 8, 3, CLK_SET_RATE_GATE); /* name parent_name base */ - clks[IMX7ULP_CLK_APLL] = imx_clk_pllv4("apll", "apll_pre_div", base + 0x500); - clks[IMX7ULP_CLK_SPLL] = imx_clk_pllv4("spll", "spll_pre_div", base + 0x600); + hws[IMX7ULP_CLK_APLL] = imx_clk_hw_pllv4("apll", "apll_pre_div", base + 0x500); + hws[IMX7ULP_CLK_SPLL] = imx_clk_hw_pllv4("spll", "spll_pre_div", base + 0x600); /* APLL PFDs */ - clks[IMX7ULP_CLK_APLL_PFD0] = imx_clk_pfdv2("apll_pfd0", "apll", base + 0x50c, 0); - clks[IMX7ULP_CLK_APLL_PFD1] = imx_clk_pfdv2("apll_pfd1", "apll", base + 0x50c, 1); - clks[IMX7ULP_CLK_APLL_PFD2] = imx_clk_pfdv2("apll_pfd2", "apll", base + 0x50c, 2); - clks[IMX7ULP_CLK_APLL_PFD3] = imx_clk_pfdv2("apll_pfd3", "apll", base + 0x50c, 3); + hws[IMX7ULP_CLK_APLL_PFD0] = imx_clk_hw_pfdv2("apll_pfd0", "apll", base + 0x50c, 0); + hws[IMX7ULP_CLK_APLL_PFD1] = imx_clk_hw_pfdv2("apll_pfd1", "apll", base + 0x50c, 1); + hws[IMX7ULP_CLK_APLL_PFD2] = imx_clk_hw_pfdv2("apll_pfd2", "apll", base + 0x50c, 2); + hws[IMX7ULP_CLK_APLL_PFD3] = imx_clk_hw_pfdv2("apll_pfd3", "apll", base + 0x50c, 3); /* SPLL PFDs */ - clks[IMX7ULP_CLK_SPLL_PFD0] = imx_clk_pfdv2("spll_pfd0", "spll", base + 0x60C, 0); - clks[IMX7ULP_CLK_SPLL_PFD1] = imx_clk_pfdv2("spll_pfd1", "spll", base + 0x60C, 1); - clks[IMX7ULP_CLK_SPLL_PFD2] = imx_clk_pfdv2("spll_pfd2", "spll", base + 0x60C, 2); - clks[IMX7ULP_CLK_SPLL_PFD3] = imx_clk_pfdv2("spll_pfd3", "spll", base + 0x60C, 3); + hws[IMX7ULP_CLK_SPLL_PFD0] = imx_clk_hw_pfdv2("spll_pfd0", "spll", base + 0x60C, 0); + hws[IMX7ULP_CLK_SPLL_PFD1] = imx_clk_hw_pfdv2("spll_pfd1", "spll", base + 0x60C, 1); + hws[IMX7ULP_CLK_SPLL_PFD2] = imx_clk_hw_pfdv2("spll_pfd2", "spll", base + 0x60C, 2); + hws[IMX7ULP_CLK_SPLL_PFD3] = imx_clk_hw_pfdv2("spll_pfd3", "spll", base + 0x60C, 3); /* PLL Mux */ - clks[IMX7ULP_CLK_APLL_PFD_SEL] = imx_clk_hw_mux_flags("apll_pfd_sel", base + 0x508, 14, 2, apll_pfd_sels, ARRAY_SIZE(apll_pfd_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); - clks[IMX7ULP_CLK_SPLL_PFD_SEL] = imx_clk_hw_mux_flags("spll_pfd_sel", base + 0x608, 14, 2, spll_pfd_sels, ARRAY_SIZE(spll_pfd_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); - clks[IMX7ULP_CLK_APLL_SEL] = imx_clk_hw_mux_flags("apll_sel", base + 0x508, 1, 1, apll_sels, ARRAY_SIZE(apll_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); - clks[IMX7ULP_CLK_SPLL_SEL] = imx_clk_hw_mux_flags("spll_sel", base + 0x608, 1, 1, spll_sels, ARRAY_SIZE(spll_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); + hws[IMX7ULP_CLK_APLL_PFD_SEL] = imx_clk_hw_mux_flags("apll_pfd_sel", base + 0x508, 14, 2, apll_pfd_sels, ARRAY_SIZE(apll_pfd_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); + hws[IMX7ULP_CLK_SPLL_PFD_SEL] = imx_clk_hw_mux_flags("spll_pfd_sel", base + 0x608, 14, 2, spll_pfd_sels, ARRAY_SIZE(spll_pfd_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); + hws[IMX7ULP_CLK_APLL_SEL] = imx_clk_hw_mux_flags("apll_sel", base + 0x508, 1, 1, apll_sels, ARRAY_SIZE(apll_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); + hws[IMX7ULP_CLK_SPLL_SEL] = imx_clk_hw_mux_flags("spll_sel", base + 0x608, 1, 1, spll_sels, ARRAY_SIZE(spll_sels), CLK_SET_RATE_PARENT | CLK_SET_PARENT_GATE); - clks[IMX7ULP_CLK_SPLL_BUS_CLK] = imx_clk_divider_gate("spll_bus_clk", "spll_sel", CLK_SET_RATE_GATE, base + 0x604, 8, 3, 0, ulp_div_table, &imx_ccm_lock); + hws[IMX7ULP_CLK_SPLL_BUS_CLK] = imx_clk_hw_divider_gate("spll_bus_clk", "spll_sel", CLK_SET_RATE_GATE, base + 0x604, 8, 3, 0, ulp_div_table, &imx_ccm_lock); /* scs/ddr/nic select different clock source requires that clock to be enabled first */ - clks[IMX7ULP_CLK_SYS_SEL] = imx_clk_hw_mux2("scs_sel", base + 0x14, 24, 4, scs_sels, ARRAY_SIZE(scs_sels)); - clks[IMX7ULP_CLK_HSRUN_SYS_SEL] = imx_clk_hw_mux2("hsrun_scs_sel", base + 0x1c, 24, 4, scs_sels, ARRAY_SIZE(scs_sels)); - clks[IMX7ULP_CLK_NIC_SEL] = imx_clk_hw_mux2("nic_sel", base + 0x40, 28, 1, nic_sels, ARRAY_SIZE(nic_sels)); - clks[IMX7ULP_CLK_DDR_SEL] = imx_clk_hw_mux_flags("ddr_sel", base + 0x30, 24, 2, ddr_sels, ARRAY_SIZE(ddr_sels), CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); + hws[IMX7ULP_CLK_SYS_SEL] = imx_clk_hw_mux2("scs_sel", base + 0x14, 24, 4, scs_sels, ARRAY_SIZE(scs_sels)); + hws[IMX7ULP_CLK_HSRUN_SYS_SEL] = imx_clk_hw_mux2("hsrun_scs_sel", base + 0x1c, 24, 4, scs_sels, ARRAY_SIZE(scs_sels)); + hws[IMX7ULP_CLK_NIC_SEL] = imx_clk_hw_mux2("nic_sel", base + 0x40, 28, 1, nic_sels, ARRAY_SIZE(nic_sels)); + hws[IMX7ULP_CLK_DDR_SEL] = imx_clk_hw_mux_flags("ddr_sel", base + 0x30, 24, 2, ddr_sels, ARRAY_SIZE(ddr_sels), CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); - clks[IMX7ULP_CLK_CORE_DIV] = imx_clk_hw_divider_flags("divcore", "scs_sel", base + 0x14, 16, 4, CLK_SET_RATE_PARENT); - clks[IMX7ULP_CLK_HSRUN_CORE_DIV] = imx_clk_hw_divider_flags("hsrun_divcore", "hsrun_scs_sel", base + 0x1c, 16, 4, CLK_SET_RATE_PARENT); + hws[IMX7ULP_CLK_CORE_DIV] = imx_clk_hw_divider_flags("divcore", "scs_sel", base + 0x14, 16, 4, CLK_SET_RATE_PARENT); + hws[IMX7ULP_CLK_HSRUN_CORE_DIV] = imx_clk_hw_divider_flags("hsrun_divcore", "hsrun_scs_sel", base + 0x1c, 16, 4, CLK_SET_RATE_PARENT); - clks[IMX7ULP_CLK_DDR_DIV] = imx_clk_divider_gate("ddr_clk", "ddr_sel", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, base + 0x30, 0, 3, + hws[IMX7ULP_CLK_DDR_DIV] = imx_clk_hw_divider_gate("ddr_clk", "ddr_sel", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, base + 0x30, 0, 3, 0, ulp_div_table, &imx_ccm_lock); - clks[IMX7ULP_CLK_NIC0_DIV] = imx_clk_hw_divider_flags("nic0_clk", "nic_sel", base + 0x40, 24, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); - clks[IMX7ULP_CLK_NIC1_DIV] = imx_clk_hw_divider_flags("nic1_clk", "nic0_clk", base + 0x40, 16, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); - clks[IMX7ULP_CLK_NIC1_BUS_DIV] = imx_clk_hw_divider_flags("nic1_bus_clk", "nic0_clk", base + 0x40, 4, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); + hws[IMX7ULP_CLK_NIC0_DIV] = imx_clk_hw_divider_flags("nic0_clk", "nic_sel", base + 0x40, 24, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); + hws[IMX7ULP_CLK_NIC1_DIV] = imx_clk_hw_divider_flags("nic1_clk", "nic0_clk", base + 0x40, 16, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); + hws[IMX7ULP_CLK_NIC1_BUS_DIV] = imx_clk_hw_divider_flags("nic1_bus_clk", "nic0_clk", base + 0x40, 4, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); - clks[IMX7ULP_CLK_GPU_DIV] = imx_clk_hw_divider("gpu_clk", "nic0_clk", base + 0x40, 20, 4); + hws[IMX7ULP_CLK_GPU_DIV] = imx_clk_hw_divider("gpu_clk", "nic0_clk", base + 0x40, 20, 4); - clks[IMX7ULP_CLK_SOSC_BUS_CLK] = imx_clk_divider_gate("sosc_bus_clk", "sosc", 0, base + 0x104, 8, 3, + hws[IMX7ULP_CLK_SOSC_BUS_CLK] = imx_clk_hw_divider_gate("sosc_bus_clk", "sosc", 0, base + 0x104, 8, 3, CLK_DIVIDER_READ_ONLY, ulp_div_table, &imx_ccm_lock); - clks[IMX7ULP_CLK_FIRC_BUS_CLK] = imx_clk_divider_gate("firc_bus_clk", "firc", 0, base + 0x304, 8, 3, + hws[IMX7ULP_CLK_FIRC_BUS_CLK] = imx_clk_hw_divider_gate("firc_bus_clk", "firc", 0, base + 0x304, 8, 3, CLK_DIVIDER_READ_ONLY, ulp_div_table, &imx_ccm_lock); - imx_check_clk_hws(clks, clk_data->num); + imx_check_clk_hws(hws, clk_data->num); of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); } @@ -146,7 +146,7 @@ CLK_OF_DECLARE(imx7ulp_clk_scg1, "fsl,imx7ulp-scg1", imx7ulp_clk_scg1_init); static void __init imx7ulp_clk_pcc2_init(struct device_node *np) { struct clk_hw_onecell_data *clk_data; - struct clk_hw **clks; + struct clk_hw **hws; void __iomem *base; int i; @@ -156,42 +156,42 @@ static void __init imx7ulp_clk_pcc2_init(struct device_node *np) return; clk_data->num = IMX7ULP_CLK_PCC2_END; - clks = clk_data->hws; + hws = clk_data->hws; /* PCC2 */ base = of_iomap(np, 0); WARN_ON(!base); - clks[IMX7ULP_CLK_DMA1] = imx_clk_hw_gate("dma1", "nic1_clk", base + 0x20, 30); - clks[IMX7ULP_CLK_RGPIO2P1] = imx_clk_hw_gate("rgpio2p1", "nic1_bus_clk", base + 0x3c, 30); - clks[IMX7ULP_CLK_DMA_MUX1] = imx_clk_hw_gate("dma_mux1", "nic1_bus_clk", base + 0x84, 30); - clks[IMX7ULP_CLK_CAAM] = imx_clk_hw_gate("caam", "nic1_clk", base + 0x90, 30); - clks[IMX7ULP_CLK_LPTPM4] = imx7ulp_clk_composite("lptpm4", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x94); - clks[IMX7ULP_CLK_LPTPM5] = imx7ulp_clk_composite("lptpm5", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x98); - clks[IMX7ULP_CLK_LPIT1] = imx7ulp_clk_composite("lpit1", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x9c); - clks[IMX7ULP_CLK_LPSPI2] = imx7ulp_clk_composite("lpspi2", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xa4); - clks[IMX7ULP_CLK_LPSPI3] = imx7ulp_clk_composite("lpspi3", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xa8); - clks[IMX7ULP_CLK_LPI2C4] = imx7ulp_clk_composite("lpi2c4", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xac); - clks[IMX7ULP_CLK_LPI2C5] = imx7ulp_clk_composite("lpi2c5", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xb0); - clks[IMX7ULP_CLK_LPUART4] = imx7ulp_clk_composite("lpuart4", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xb4); - clks[IMX7ULP_CLK_LPUART5] = imx7ulp_clk_composite("lpuart5", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xb8); - clks[IMX7ULP_CLK_FLEXIO1] = imx7ulp_clk_composite("flexio1", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xc4); - clks[IMX7ULP_CLK_USB0] = imx7ulp_clk_composite("usb0", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xcc); - clks[IMX7ULP_CLK_USB1] = imx7ulp_clk_composite("usb1", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xd0); - clks[IMX7ULP_CLK_USB_PHY] = imx_clk_hw_gate("usb_phy", "nic1_bus_clk", base + 0xd4, 30); - clks[IMX7ULP_CLK_USDHC0] = imx7ulp_clk_composite("usdhc0", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xdc); - clks[IMX7ULP_CLK_USDHC1] = imx7ulp_clk_composite("usdhc1", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xe0); - clks[IMX7ULP_CLK_WDG1] = imx7ulp_clk_composite("wdg1", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, true, true, base + 0xf4); - clks[IMX7ULP_CLK_WDG2] = imx7ulp_clk_composite("sdg2", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, true, true, base + 0x10c); - - imx_check_clk_hws(clks, clk_data->num); + hws[IMX7ULP_CLK_DMA1] = imx_clk_hw_gate("dma1", "nic1_clk", base + 0x20, 30); + hws[IMX7ULP_CLK_RGPIO2P1] = imx_clk_hw_gate("rgpio2p1", "nic1_bus_clk", base + 0x3c, 30); + hws[IMX7ULP_CLK_DMA_MUX1] = imx_clk_hw_gate("dma_mux1", "nic1_bus_clk", base + 0x84, 30); + hws[IMX7ULP_CLK_CAAM] = imx_clk_hw_gate("caam", "nic1_clk", base + 0x90, 30); + hws[IMX7ULP_CLK_LPTPM4] = imx7ulp_clk_hw_composite("lptpm4", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x94); + hws[IMX7ULP_CLK_LPTPM5] = imx7ulp_clk_hw_composite("lptpm5", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x98); + hws[IMX7ULP_CLK_LPIT1] = imx7ulp_clk_hw_composite("lpit1", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x9c); + hws[IMX7ULP_CLK_LPSPI2] = imx7ulp_clk_hw_composite("lpspi2", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xa4); + hws[IMX7ULP_CLK_LPSPI3] = imx7ulp_clk_hw_composite("lpspi3", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xa8); + hws[IMX7ULP_CLK_LPI2C4] = imx7ulp_clk_hw_composite("lpi2c4", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xac); + hws[IMX7ULP_CLK_LPI2C5] = imx7ulp_clk_hw_composite("lpi2c5", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xb0); + hws[IMX7ULP_CLK_LPUART4] = imx7ulp_clk_hw_composite("lpuart4", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xb4); + hws[IMX7ULP_CLK_LPUART5] = imx7ulp_clk_hw_composite("lpuart5", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xb8); + hws[IMX7ULP_CLK_FLEXIO1] = imx7ulp_clk_hw_composite("flexio1", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0xc4); + hws[IMX7ULP_CLK_USB0] = imx7ulp_clk_hw_composite("usb0", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xcc); + hws[IMX7ULP_CLK_USB1] = imx7ulp_clk_hw_composite("usb1", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xd0); + hws[IMX7ULP_CLK_USB_PHY] = imx_clk_hw_gate("usb_phy", "nic1_bus_clk", base + 0xd4, 30); + hws[IMX7ULP_CLK_USDHC0] = imx7ulp_clk_hw_composite("usdhc0", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xdc); + hws[IMX7ULP_CLK_USDHC1] = imx7ulp_clk_hw_composite("usdhc1", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xe0); + hws[IMX7ULP_CLK_WDG1] = imx7ulp_clk_hw_composite("wdg1", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, true, true, base + 0xf4); + hws[IMX7ULP_CLK_WDG2] = imx7ulp_clk_hw_composite("wdg2", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, true, true, base + 0x10c); + + imx_check_clk_hws(hws, clk_data->num); of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); for (i = 0; i < ARRAY_SIZE(pcc2_uart_clk_ids); i++) { int index = pcc2_uart_clk_ids[i]; - pcc2_uart_clks[i] = &clks[index]->clk; + pcc2_uart_clks[i] = &hws[index]->clk; } imx_register_uart_clocks(pcc2_uart_clks); @@ -201,7 +201,7 @@ CLK_OF_DECLARE(imx7ulp_clk_pcc2, "fsl,imx7ulp-pcc2", imx7ulp_clk_pcc2_init); static void __init imx7ulp_clk_pcc3_init(struct device_node *np) { struct clk_hw_onecell_data *clk_data; - struct clk_hw **clks; + struct clk_hw **hws; void __iomem *base; int i; @@ -211,41 +211,41 @@ static void __init imx7ulp_clk_pcc3_init(struct device_node *np) return; clk_data->num = IMX7ULP_CLK_PCC3_END; - clks = clk_data->hws; + hws = clk_data->hws; /* PCC3 */ base = of_iomap(np, 0); WARN_ON(!base); - clks[IMX7ULP_CLK_LPTPM6] = imx7ulp_clk_composite("lptpm6", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x84); - clks[IMX7ULP_CLK_LPTPM7] = imx7ulp_clk_composite("lptpm7", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x88); + hws[IMX7ULP_CLK_LPTPM6] = imx7ulp_clk_hw_composite("lptpm6", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x84); + hws[IMX7ULP_CLK_LPTPM7] = imx7ulp_clk_hw_composite("lptpm7", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x88); - clks[IMX7ULP_CLK_MMDC] = clk_hw_register_gate(NULL, "mmdc", "nic1_clk", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, + hws[IMX7ULP_CLK_MMDC] = clk_hw_register_gate(NULL, "mmdc", "nic1_clk", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, base + 0xac, 30, 0, &imx_ccm_lock); - clks[IMX7ULP_CLK_LPI2C6] = imx7ulp_clk_composite("lpi2c6", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x90); - clks[IMX7ULP_CLK_LPI2C7] = imx7ulp_clk_composite("lpi2c7", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x94); - clks[IMX7ULP_CLK_LPUART6] = imx7ulp_clk_composite("lpuart6", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x98); - clks[IMX7ULP_CLK_LPUART7] = imx7ulp_clk_composite("lpuart7", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x9c); - clks[IMX7ULP_CLK_DSI] = imx7ulp_clk_composite("dsi", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, true, true, base + 0xa4); - clks[IMX7ULP_CLK_LCDIF] = imx7ulp_clk_composite("lcdif", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xa8); + hws[IMX7ULP_CLK_LPI2C6] = imx7ulp_clk_hw_composite("lpi2c6", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x90); + hws[IMX7ULP_CLK_LPI2C7] = imx7ulp_clk_hw_composite("lpi2c7", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x94); + hws[IMX7ULP_CLK_LPUART6] = imx7ulp_clk_hw_composite("lpuart6", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x98); + hws[IMX7ULP_CLK_LPUART7] = imx7ulp_clk_hw_composite("lpuart7", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x9c); + hws[IMX7ULP_CLK_DSI] = imx7ulp_clk_hw_composite("dsi", periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, true, true, base + 0xa4); + hws[IMX7ULP_CLK_LCDIF] = imx7ulp_clk_hw_composite("lcdif", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, true, true, base + 0xa8); - clks[IMX7ULP_CLK_VIU] = imx_clk_hw_gate("viu", "nic1_clk", base + 0xa0, 30); - clks[IMX7ULP_CLK_PCTLC] = imx_clk_hw_gate("pctlc", "nic1_bus_clk", base + 0xb8, 30); - clks[IMX7ULP_CLK_PCTLD] = imx_clk_hw_gate("pctld", "nic1_bus_clk", base + 0xbc, 30); - clks[IMX7ULP_CLK_PCTLE] = imx_clk_hw_gate("pctle", "nic1_bus_clk", base + 0xc0, 30); - clks[IMX7ULP_CLK_PCTLF] = imx_clk_hw_gate("pctlf", "nic1_bus_clk", base + 0xc4, 30); + hws[IMX7ULP_CLK_VIU] = imx_clk_hw_gate("viu", "nic1_clk", base + 0xa0, 30); + hws[IMX7ULP_CLK_PCTLC] = imx_clk_hw_gate("pctlc", "nic1_bus_clk", base + 0xb8, 30); + hws[IMX7ULP_CLK_PCTLD] = imx_clk_hw_gate("pctld", "nic1_bus_clk", base + 0xbc, 30); + hws[IMX7ULP_CLK_PCTLE] = imx_clk_hw_gate("pctle", "nic1_bus_clk", base + 0xc0, 30); + hws[IMX7ULP_CLK_PCTLF] = imx_clk_hw_gate("pctlf", "nic1_bus_clk", base + 0xc4, 30); - clks[IMX7ULP_CLK_GPU3D] = imx7ulp_clk_composite("gpu3d", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, false, true, base + 0x140); - clks[IMX7ULP_CLK_GPU2D] = imx7ulp_clk_composite("gpu2d", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, false, true, base + 0x144); + hws[IMX7ULP_CLK_GPU3D] = imx7ulp_clk_hw_composite("gpu3d", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, false, true, base + 0x140); + hws[IMX7ULP_CLK_GPU2D] = imx7ulp_clk_hw_composite("gpu2d", periph_plat_sels, ARRAY_SIZE(periph_plat_sels), true, false, true, base + 0x144); - imx_check_clk_hws(clks, clk_data->num); + imx_check_clk_hws(hws, clk_data->num); of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); for (i = 0; i < ARRAY_SIZE(pcc3_uart_clk_ids); i++) { int index = pcc3_uart_clk_ids[i]; - pcc3_uart_clks[i] = &clks[index]->clk; + pcc3_uart_clks[i] = &hws[index]->clk; } imx_register_uart_clocks(pcc3_uart_clks); @@ -255,7 +255,7 @@ CLK_OF_DECLARE(imx7ulp_clk_pcc3, "fsl,imx7ulp-pcc3", imx7ulp_clk_pcc3_init); static void __init imx7ulp_clk_smc1_init(struct device_node *np) { struct clk_hw_onecell_data *clk_data; - struct clk_hw **clks; + struct clk_hw **hws; void __iomem *base; clk_data = kzalloc(struct_size(clk_data, hws, IMX7ULP_CLK_SMC1_END), @@ -264,15 +264,15 @@ static void __init imx7ulp_clk_smc1_init(struct device_node *np) return; clk_data->num = IMX7ULP_CLK_SMC1_END; - clks = clk_data->hws; + hws = clk_data->hws; /* SMC1 */ base = of_iomap(np, 0); WARN_ON(!base); - clks[IMX7ULP_CLK_ARM] = imx_clk_hw_mux_flags("arm", base + 0x10, 8, 2, arm_sels, ARRAY_SIZE(arm_sels), CLK_IS_CRITICAL); + hws[IMX7ULP_CLK_ARM] = imx_clk_hw_mux_flags("arm", base + 0x10, 8, 2, arm_sels, ARRAY_SIZE(arm_sels), CLK_IS_CRITICAL); - imx_check_clk_hws(clks, clk_data->num); + imx_check_clk_hws(hws, clk_data->num); of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); } diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c index 030b15d7c0ce..2ed93fc25087 100644 --- a/drivers/clk/imx/clk-imx8mm.c +++ b/drivers/clk/imx/clk-imx8mm.c @@ -12,6 +12,7 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/platform_device.h> +#include <linux/slab.h> #include <linux/types.h> #include "clk.h" @@ -285,118 +286,126 @@ static const char *imx8mm_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", } static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m", "sys_pll1_200m", "audio_pll2_out", "vpu_pll", "sys_pll1_80m", }; -static struct clk *clks[IMX8MM_CLK_END]; -static struct clk_onecell_data clk_data; +static struct clk_hw_onecell_data *clk_hw_data; +static struct clk_hw **hws; -static struct clk ** const uart_clks[] = { - &clks[IMX8MM_CLK_UART1_ROOT], - &clks[IMX8MM_CLK_UART2_ROOT], - &clks[IMX8MM_CLK_UART3_ROOT], - &clks[IMX8MM_CLK_UART4_ROOT], - NULL +static const int uart_clk_ids[] = { + IMX8MM_CLK_UART1_ROOT, + IMX8MM_CLK_UART2_ROOT, + IMX8MM_CLK_UART3_ROOT, + IMX8MM_CLK_UART4_ROOT, }; +static struct clk **uart_hws[ARRAY_SIZE(uart_clk_ids) + 1]; static int imx8mm_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; void __iomem *base; - int ret; + int ret, i; - clks[IMX8MM_CLK_DUMMY] = imx_clk_fixed("dummy", 0); - clks[IMX8MM_CLK_24M] = of_clk_get_by_name(np, "osc_24m"); - clks[IMX8MM_CLK_32K] = of_clk_get_by_name(np, "osc_32k"); - clks[IMX8MM_CLK_EXT1] = of_clk_get_by_name(np, "clk_ext1"); - clks[IMX8MM_CLK_EXT2] = of_clk_get_by_name(np, "clk_ext2"); - clks[IMX8MM_CLK_EXT3] = of_clk_get_by_name(np, "clk_ext3"); - clks[IMX8MM_CLK_EXT4] = of_clk_get_by_name(np, "clk_ext4"); + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX8MM_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return -ENOMEM; + + clk_hw_data->num = IMX8MM_CLK_END; + hws = clk_hw_data->hws; + + hws[IMX8MM_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); + hws[IMX8MM_CLK_24M] = imx_obtain_fixed_clk_hw(np, "osc_24m"); + hws[IMX8MM_CLK_32K] = imx_obtain_fixed_clk_hw(np, "osc_32k"); + hws[IMX8MM_CLK_EXT1] = imx_obtain_fixed_clk_hw(np, "clk_ext1"); + hws[IMX8MM_CLK_EXT2] = imx_obtain_fixed_clk_hw(np, "clk_ext2"); + hws[IMX8MM_CLK_EXT3] = imx_obtain_fixed_clk_hw(np, "clk_ext3"); + hws[IMX8MM_CLK_EXT4] = imx_obtain_fixed_clk_hw(np, "clk_ext4"); np = of_find_compatible_node(NULL, NULL, "fsl,imx8mm-anatop"); base = of_iomap(np, 0); if (WARN_ON(!base)) return -ENOMEM; - clks[IMX8MM_AUDIO_PLL1_REF_SEL] = imx_clk_mux("audio_pll1_ref_sel", base + 0x0, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MM_AUDIO_PLL2_REF_SEL] = imx_clk_mux("audio_pll2_ref_sel", base + 0x14, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MM_VIDEO_PLL1_REF_SEL] = imx_clk_mux("video_pll1_ref_sel", base + 0x28, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MM_DRAM_PLL_REF_SEL] = imx_clk_mux("dram_pll_ref_sel", base + 0x50, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MM_GPU_PLL_REF_SEL] = imx_clk_mux("gpu_pll_ref_sel", base + 0x64, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MM_VPU_PLL_REF_SEL] = imx_clk_mux("vpu_pll_ref_sel", base + 0x74, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MM_ARM_PLL_REF_SEL] = imx_clk_mux("arm_pll_ref_sel", base + 0x84, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MM_SYS_PLL3_REF_SEL] = imx_clk_mux("sys_pll3_ref_sel", base + 0x114, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - - clks[IMX8MM_AUDIO_PLL1] = imx_clk_pll14xx("audio_pll1", "audio_pll1_ref_sel", base, &imx_1443x_pll); - clks[IMX8MM_AUDIO_PLL2] = imx_clk_pll14xx("audio_pll2", "audio_pll2_ref_sel", base + 0x14, &imx_1443x_pll); - clks[IMX8MM_VIDEO_PLL1] = imx_clk_pll14xx("video_pll1", "video_pll1_ref_sel", base + 0x28, &imx_1443x_pll); - clks[IMX8MM_DRAM_PLL] = imx_clk_pll14xx("dram_pll", "dram_pll_ref_sel", base + 0x50, &imx_1443x_pll); - clks[IMX8MM_GPU_PLL] = imx_clk_pll14xx("gpu_pll", "gpu_pll_ref_sel", base + 0x64, &imx_1416x_pll); - clks[IMX8MM_VPU_PLL] = imx_clk_pll14xx("vpu_pll", "vpu_pll_ref_sel", base + 0x74, &imx_1416x_pll); - clks[IMX8MM_ARM_PLL] = imx_clk_pll14xx("arm_pll", "arm_pll_ref_sel", base + 0x84, &imx_1416x_pll); - clks[IMX8MM_SYS_PLL1] = imx_clk_fixed("sys_pll1", 800000000); - clks[IMX8MM_SYS_PLL2] = imx_clk_fixed("sys_pll2", 1000000000); - clks[IMX8MM_SYS_PLL3] = imx_clk_pll14xx("sys_pll3", "sys_pll3_ref_sel", base + 0x114, &imx_1416x_pll); + hws[IMX8MM_AUDIO_PLL1_REF_SEL] = imx_clk_hw_mux("audio_pll1_ref_sel", base + 0x0, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MM_AUDIO_PLL2_REF_SEL] = imx_clk_hw_mux("audio_pll2_ref_sel", base + 0x14, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MM_VIDEO_PLL1_REF_SEL] = imx_clk_hw_mux("video_pll1_ref_sel", base + 0x28, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MM_DRAM_PLL_REF_SEL] = imx_clk_hw_mux("dram_pll_ref_sel", base + 0x50, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MM_GPU_PLL_REF_SEL] = imx_clk_hw_mux("gpu_pll_ref_sel", base + 0x64, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MM_VPU_PLL_REF_SEL] = imx_clk_hw_mux("vpu_pll_ref_sel", base + 0x74, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MM_ARM_PLL_REF_SEL] = imx_clk_hw_mux("arm_pll_ref_sel", base + 0x84, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MM_SYS_PLL3_REF_SEL] = imx_clk_hw_mux("sys_pll3_ref_sel", base + 0x114, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + + hws[IMX8MM_AUDIO_PLL1] = imx_clk_hw_pll14xx("audio_pll1", "audio_pll1_ref_sel", base, &imx_1443x_pll); + hws[IMX8MM_AUDIO_PLL2] = imx_clk_hw_pll14xx("audio_pll2", "audio_pll2_ref_sel", base + 0x14, &imx_1443x_pll); + hws[IMX8MM_VIDEO_PLL1] = imx_clk_hw_pll14xx("video_pll1", "video_pll1_ref_sel", base + 0x28, &imx_1443x_pll); + hws[IMX8MM_DRAM_PLL] = imx_clk_hw_pll14xx("dram_pll", "dram_pll_ref_sel", base + 0x50, &imx_1443x_dram_pll); + hws[IMX8MM_GPU_PLL] = imx_clk_hw_pll14xx("gpu_pll", "gpu_pll_ref_sel", base + 0x64, &imx_1416x_pll); + hws[IMX8MM_VPU_PLL] = imx_clk_hw_pll14xx("vpu_pll", "vpu_pll_ref_sel", base + 0x74, &imx_1416x_pll); + hws[IMX8MM_ARM_PLL] = imx_clk_hw_pll14xx("arm_pll", "arm_pll_ref_sel", base + 0x84, &imx_1416x_pll); + hws[IMX8MM_SYS_PLL1] = imx_clk_hw_fixed("sys_pll1", 800000000); + hws[IMX8MM_SYS_PLL2] = imx_clk_hw_fixed("sys_pll2", 1000000000); + hws[IMX8MM_SYS_PLL3] = imx_clk_hw_pll14xx("sys_pll3", "sys_pll3_ref_sel", base + 0x114, &imx_1416x_pll); /* PLL bypass out */ - clks[IMX8MM_AUDIO_PLL1_BYPASS] = imx_clk_mux_flags("audio_pll1_bypass", base, 16, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MM_AUDIO_PLL2_BYPASS] = imx_clk_mux_flags("audio_pll2_bypass", base + 0x14, 16, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MM_VIDEO_PLL1_BYPASS] = imx_clk_mux_flags("video_pll1_bypass", base + 0x28, 16, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MM_DRAM_PLL_BYPASS] = imx_clk_mux_flags("dram_pll_bypass", base + 0x50, 16, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MM_GPU_PLL_BYPASS] = imx_clk_mux_flags("gpu_pll_bypass", base + 0x64, 28, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MM_VPU_PLL_BYPASS] = imx_clk_mux_flags("vpu_pll_bypass", base + 0x74, 28, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MM_ARM_PLL_BYPASS] = imx_clk_mux_flags("arm_pll_bypass", base + 0x84, 28, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MM_SYS_PLL3_BYPASS] = imx_clk_mux_flags("sys_pll3_bypass", base + 0x114, 28, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_AUDIO_PLL1_BYPASS] = imx_clk_hw_mux_flags("audio_pll1_bypass", base, 16, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_AUDIO_PLL2_BYPASS] = imx_clk_hw_mux_flags("audio_pll2_bypass", base + 0x14, 16, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_VIDEO_PLL1_BYPASS] = imx_clk_hw_mux_flags("video_pll1_bypass", base + 0x28, 16, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_DRAM_PLL_BYPASS] = imx_clk_hw_mux_flags("dram_pll_bypass", base + 0x50, 16, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_GPU_PLL_BYPASS] = imx_clk_hw_mux_flags("gpu_pll_bypass", base + 0x64, 28, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_VPU_PLL_BYPASS] = imx_clk_hw_mux_flags("vpu_pll_bypass", base + 0x74, 28, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_ARM_PLL_BYPASS] = imx_clk_hw_mux_flags("arm_pll_bypass", base + 0x84, 28, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MM_SYS_PLL3_BYPASS] = imx_clk_hw_mux_flags("sys_pll3_bypass", base + 0x114, 28, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT); /* PLL out gate */ - clks[IMX8MM_AUDIO_PLL1_OUT] = imx_clk_gate("audio_pll1_out", "audio_pll1_bypass", base, 13); - clks[IMX8MM_AUDIO_PLL2_OUT] = imx_clk_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x14, 13); - clks[IMX8MM_VIDEO_PLL1_OUT] = imx_clk_gate("video_pll1_out", "video_pll1_bypass", base + 0x28, 13); - clks[IMX8MM_DRAM_PLL_OUT] = imx_clk_gate("dram_pll_out", "dram_pll_bypass", base + 0x50, 13); - clks[IMX8MM_GPU_PLL_OUT] = imx_clk_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x64, 11); - clks[IMX8MM_VPU_PLL_OUT] = imx_clk_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x74, 11); - clks[IMX8MM_ARM_PLL_OUT] = imx_clk_gate("arm_pll_out", "arm_pll_bypass", base + 0x84, 11); - clks[IMX8MM_SYS_PLL3_OUT] = imx_clk_gate("sys_pll3_out", "sys_pll3_bypass", base + 0x114, 11); + hws[IMX8MM_AUDIO_PLL1_OUT] = imx_clk_hw_gate("audio_pll1_out", "audio_pll1_bypass", base, 13); + hws[IMX8MM_AUDIO_PLL2_OUT] = imx_clk_hw_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x14, 13); + hws[IMX8MM_VIDEO_PLL1_OUT] = imx_clk_hw_gate("video_pll1_out", "video_pll1_bypass", base + 0x28, 13); + hws[IMX8MM_DRAM_PLL_OUT] = imx_clk_hw_gate("dram_pll_out", "dram_pll_bypass", base + 0x50, 13); + hws[IMX8MM_GPU_PLL_OUT] = imx_clk_hw_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x64, 11); + hws[IMX8MM_VPU_PLL_OUT] = imx_clk_hw_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x74, 11); + hws[IMX8MM_ARM_PLL_OUT] = imx_clk_hw_gate("arm_pll_out", "arm_pll_bypass", base + 0x84, 11); + hws[IMX8MM_SYS_PLL3_OUT] = imx_clk_hw_gate("sys_pll3_out", "sys_pll3_bypass", base + 0x114, 11); /* SYS PLL1 fixed output */ - clks[IMX8MM_SYS_PLL1_40M_CG] = imx_clk_gate("sys_pll1_40m_cg", "sys_pll1", base + 0x94, 27); - clks[IMX8MM_SYS_PLL1_80M_CG] = imx_clk_gate("sys_pll1_80m_cg", "sys_pll1", base + 0x94, 25); - clks[IMX8MM_SYS_PLL1_100M_CG] = imx_clk_gate("sys_pll1_100m_cg", "sys_pll1", base + 0x94, 23); - clks[IMX8MM_SYS_PLL1_133M_CG] = imx_clk_gate("sys_pll1_133m_cg", "sys_pll1", base + 0x94, 21); - clks[IMX8MM_SYS_PLL1_160M_CG] = imx_clk_gate("sys_pll1_160m_cg", "sys_pll1", base + 0x94, 19); - clks[IMX8MM_SYS_PLL1_200M_CG] = imx_clk_gate("sys_pll1_200m_cg", "sys_pll1", base + 0x94, 17); - clks[IMX8MM_SYS_PLL1_266M_CG] = imx_clk_gate("sys_pll1_266m_cg", "sys_pll1", base + 0x94, 15); - clks[IMX8MM_SYS_PLL1_400M_CG] = imx_clk_gate("sys_pll1_400m_cg", "sys_pll1", base + 0x94, 13); - clks[IMX8MM_SYS_PLL1_OUT] = imx_clk_gate("sys_pll1_out", "sys_pll1", base + 0x94, 11); - - clks[IMX8MM_SYS_PLL1_40M] = imx_clk_fixed_factor("sys_pll1_40m", "sys_pll1_40m_cg", 1, 20); - clks[IMX8MM_SYS_PLL1_80M] = imx_clk_fixed_factor("sys_pll1_80m", "sys_pll1_80m_cg", 1, 10); - clks[IMX8MM_SYS_PLL1_100M] = imx_clk_fixed_factor("sys_pll1_100m", "sys_pll1_100m_cg", 1, 8); - clks[IMX8MM_SYS_PLL1_133M] = imx_clk_fixed_factor("sys_pll1_133m", "sys_pll1_133m_cg", 1, 6); - clks[IMX8MM_SYS_PLL1_160M] = imx_clk_fixed_factor("sys_pll1_160m", "sys_pll1_160m_cg", 1, 5); - clks[IMX8MM_SYS_PLL1_200M] = imx_clk_fixed_factor("sys_pll1_200m", "sys_pll1_200m_cg", 1, 4); - clks[IMX8MM_SYS_PLL1_266M] = imx_clk_fixed_factor("sys_pll1_266m", "sys_pll1_266m_cg", 1, 3); - clks[IMX8MM_SYS_PLL1_400M] = imx_clk_fixed_factor("sys_pll1_400m", "sys_pll1_400m_cg", 1, 2); - clks[IMX8MM_SYS_PLL1_800M] = imx_clk_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1); + hws[IMX8MM_SYS_PLL1_40M_CG] = imx_clk_hw_gate("sys_pll1_40m_cg", "sys_pll1", base + 0x94, 27); + hws[IMX8MM_SYS_PLL1_80M_CG] = imx_clk_hw_gate("sys_pll1_80m_cg", "sys_pll1", base + 0x94, 25); + hws[IMX8MM_SYS_PLL1_100M_CG] = imx_clk_hw_gate("sys_pll1_100m_cg", "sys_pll1", base + 0x94, 23); + hws[IMX8MM_SYS_PLL1_133M_CG] = imx_clk_hw_gate("sys_pll1_133m_cg", "sys_pll1", base + 0x94, 21); + hws[IMX8MM_SYS_PLL1_160M_CG] = imx_clk_hw_gate("sys_pll1_160m_cg", "sys_pll1", base + 0x94, 19); + hws[IMX8MM_SYS_PLL1_200M_CG] = imx_clk_hw_gate("sys_pll1_200m_cg", "sys_pll1", base + 0x94, 17); + hws[IMX8MM_SYS_PLL1_266M_CG] = imx_clk_hw_gate("sys_pll1_266m_cg", "sys_pll1", base + 0x94, 15); + hws[IMX8MM_SYS_PLL1_400M_CG] = imx_clk_hw_gate("sys_pll1_400m_cg", "sys_pll1", base + 0x94, 13); + hws[IMX8MM_SYS_PLL1_OUT] = imx_clk_hw_gate("sys_pll1_out", "sys_pll1", base + 0x94, 11); + + hws[IMX8MM_SYS_PLL1_40M] = imx_clk_hw_fixed_factor("sys_pll1_40m", "sys_pll1_40m_cg", 1, 20); + hws[IMX8MM_SYS_PLL1_80M] = imx_clk_hw_fixed_factor("sys_pll1_80m", "sys_pll1_80m_cg", 1, 10); + hws[IMX8MM_SYS_PLL1_100M] = imx_clk_hw_fixed_factor("sys_pll1_100m", "sys_pll1_100m_cg", 1, 8); + hws[IMX8MM_SYS_PLL1_133M] = imx_clk_hw_fixed_factor("sys_pll1_133m", "sys_pll1_133m_cg", 1, 6); + hws[IMX8MM_SYS_PLL1_160M] = imx_clk_hw_fixed_factor("sys_pll1_160m", "sys_pll1_160m_cg", 1, 5); + hws[IMX8MM_SYS_PLL1_200M] = imx_clk_hw_fixed_factor("sys_pll1_200m", "sys_pll1_200m_cg", 1, 4); + hws[IMX8MM_SYS_PLL1_266M] = imx_clk_hw_fixed_factor("sys_pll1_266m", "sys_pll1_266m_cg", 1, 3); + hws[IMX8MM_SYS_PLL1_400M] = imx_clk_hw_fixed_factor("sys_pll1_400m", "sys_pll1_400m_cg", 1, 2); + hws[IMX8MM_SYS_PLL1_800M] = imx_clk_hw_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1); /* SYS PLL2 fixed output */ - clks[IMX8MM_SYS_PLL2_50M_CG] = imx_clk_gate("sys_pll2_50m_cg", "sys_pll2", base + 0x104, 27); - clks[IMX8MM_SYS_PLL2_100M_CG] = imx_clk_gate("sys_pll2_100m_cg", "sys_pll2", base + 0x104, 25); - clks[IMX8MM_SYS_PLL2_125M_CG] = imx_clk_gate("sys_pll2_125m_cg", "sys_pll2", base + 0x104, 23); - clks[IMX8MM_SYS_PLL2_166M_CG] = imx_clk_gate("sys_pll2_166m_cg", "sys_pll2", base + 0x104, 21); - clks[IMX8MM_SYS_PLL2_200M_CG] = imx_clk_gate("sys_pll2_200m_cg", "sys_pll2", base + 0x104, 19); - clks[IMX8MM_SYS_PLL2_250M_CG] = imx_clk_gate("sys_pll2_250m_cg", "sys_pll2", base + 0x104, 17); - clks[IMX8MM_SYS_PLL2_333M_CG] = imx_clk_gate("sys_pll2_333m_cg", "sys_pll2", base + 0x104, 15); - clks[IMX8MM_SYS_PLL2_500M_CG] = imx_clk_gate("sys_pll2_500m_cg", "sys_pll2", base + 0x104, 13); - clks[IMX8MM_SYS_PLL2_OUT] = imx_clk_gate("sys_pll2_out", "sys_pll2", base + 0x104, 11); - - clks[IMX8MM_SYS_PLL2_50M] = imx_clk_fixed_factor("sys_pll2_50m", "sys_pll2_50m_cg", 1, 20); - clks[IMX8MM_SYS_PLL2_100M] = imx_clk_fixed_factor("sys_pll2_100m", "sys_pll2_100m_cg", 1, 10); - clks[IMX8MM_SYS_PLL2_125M] = imx_clk_fixed_factor("sys_pll2_125m", "sys_pll2_125m_cg", 1, 8); - clks[IMX8MM_SYS_PLL2_166M] = imx_clk_fixed_factor("sys_pll2_166m", "sys_pll2_166m_cg", 1, 6); - clks[IMX8MM_SYS_PLL2_200M] = imx_clk_fixed_factor("sys_pll2_200m", "sys_pll2_200m_cg", 1, 5); - clks[IMX8MM_SYS_PLL2_250M] = imx_clk_fixed_factor("sys_pll2_250m", "sys_pll2_250m_cg", 1, 4); - clks[IMX8MM_SYS_PLL2_333M] = imx_clk_fixed_factor("sys_pll2_333m", "sys_pll2_333m_cg", 1, 3); - clks[IMX8MM_SYS_PLL2_500M] = imx_clk_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); - clks[IMX8MM_SYS_PLL2_1000M] = imx_clk_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + hws[IMX8MM_SYS_PLL2_50M_CG] = imx_clk_hw_gate("sys_pll2_50m_cg", "sys_pll2", base + 0x104, 27); + hws[IMX8MM_SYS_PLL2_100M_CG] = imx_clk_hw_gate("sys_pll2_100m_cg", "sys_pll2", base + 0x104, 25); + hws[IMX8MM_SYS_PLL2_125M_CG] = imx_clk_hw_gate("sys_pll2_125m_cg", "sys_pll2", base + 0x104, 23); + hws[IMX8MM_SYS_PLL2_166M_CG] = imx_clk_hw_gate("sys_pll2_166m_cg", "sys_pll2", base + 0x104, 21); + hws[IMX8MM_SYS_PLL2_200M_CG] = imx_clk_hw_gate("sys_pll2_200m_cg", "sys_pll2", base + 0x104, 19); + hws[IMX8MM_SYS_PLL2_250M_CG] = imx_clk_hw_gate("sys_pll2_250m_cg", "sys_pll2", base + 0x104, 17); + hws[IMX8MM_SYS_PLL2_333M_CG] = imx_clk_hw_gate("sys_pll2_333m_cg", "sys_pll2", base + 0x104, 15); + hws[IMX8MM_SYS_PLL2_500M_CG] = imx_clk_hw_gate("sys_pll2_500m_cg", "sys_pll2", base + 0x104, 13); + hws[IMX8MM_SYS_PLL2_OUT] = imx_clk_hw_gate("sys_pll2_out", "sys_pll2", base + 0x104, 11); + + hws[IMX8MM_SYS_PLL2_50M] = imx_clk_hw_fixed_factor("sys_pll2_50m", "sys_pll2_50m_cg", 1, 20); + hws[IMX8MM_SYS_PLL2_100M] = imx_clk_hw_fixed_factor("sys_pll2_100m", "sys_pll2_100m_cg", 1, 10); + hws[IMX8MM_SYS_PLL2_125M] = imx_clk_hw_fixed_factor("sys_pll2_125m", "sys_pll2_125m_cg", 1, 8); + hws[IMX8MM_SYS_PLL2_166M] = imx_clk_hw_fixed_factor("sys_pll2_166m", "sys_pll2_166m_cg", 1, 6); + hws[IMX8MM_SYS_PLL2_200M] = imx_clk_hw_fixed_factor("sys_pll2_200m", "sys_pll2_200m_cg", 1, 5); + hws[IMX8MM_SYS_PLL2_250M] = imx_clk_hw_fixed_factor("sys_pll2_250m", "sys_pll2_250m_cg", 1, 4); + hws[IMX8MM_SYS_PLL2_333M] = imx_clk_hw_fixed_factor("sys_pll2_333m", "sys_pll2_333m_cg", 1, 3); + hws[IMX8MM_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); + hws[IMX8MM_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); @@ -404,204 +413,213 @@ static int imx8mm_clocks_probe(struct platform_device *pdev) return PTR_ERR(base); /* Core Slice */ - clks[IMX8MM_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mm_a53_sels, ARRAY_SIZE(imx8mm_a53_sels)); - clks[IMX8MM_CLK_M4_SRC] = imx_clk_mux2("arm_m4_src", base + 0x8080, 24, 3, imx8mm_m4_sels, ARRAY_SIZE(imx8mm_m4_sels)); - clks[IMX8MM_CLK_VPU_SRC] = imx_clk_mux2("vpu_src", base + 0x8100, 24, 3, imx8mm_vpu_sels, ARRAY_SIZE(imx8mm_vpu_sels)); - clks[IMX8MM_CLK_GPU3D_SRC] = imx_clk_mux2("gpu3d_src", base + 0x8180, 24, 3, imx8mm_gpu3d_sels, ARRAY_SIZE(imx8mm_gpu3d_sels)); - clks[IMX8MM_CLK_GPU2D_SRC] = imx_clk_mux2("gpu2d_src", base + 0x8200, 24, 3, imx8mm_gpu2d_sels, ARRAY_SIZE(imx8mm_gpu2d_sels)); - clks[IMX8MM_CLK_A53_CG] = imx_clk_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28); - clks[IMX8MM_CLK_M4_CG] = imx_clk_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); - clks[IMX8MM_CLK_VPU_CG] = imx_clk_gate3("vpu_cg", "vpu_src", base + 0x8100, 28); - clks[IMX8MM_CLK_GPU3D_CG] = imx_clk_gate3("gpu3d_cg", "gpu3d_src", base + 0x8180, 28); - clks[IMX8MM_CLK_GPU2D_CG] = imx_clk_gate3("gpu2d_cg", "gpu2d_src", base + 0x8200, 28); - clks[IMX8MM_CLK_A53_DIV] = imx_clk_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); - clks[IMX8MM_CLK_M4_DIV] = imx_clk_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); - clks[IMX8MM_CLK_VPU_DIV] = imx_clk_divider2("vpu_div", "vpu_cg", base + 0x8100, 0, 3); - clks[IMX8MM_CLK_GPU3D_DIV] = imx_clk_divider2("gpu3d_div", "gpu3d_cg", base + 0x8180, 0, 3); - clks[IMX8MM_CLK_GPU2D_DIV] = imx_clk_divider2("gpu2d_div", "gpu2d_cg", base + 0x8200, 0, 3); + hws[IMX8MM_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mm_a53_sels, ARRAY_SIZE(imx8mm_a53_sels)); + hws[IMX8MM_CLK_M4_SRC] = imx_clk_hw_mux2("arm_m4_src", base + 0x8080, 24, 3, imx8mm_m4_sels, ARRAY_SIZE(imx8mm_m4_sels)); + hws[IMX8MM_CLK_VPU_SRC] = imx_clk_hw_mux2("vpu_src", base + 0x8100, 24, 3, imx8mm_vpu_sels, ARRAY_SIZE(imx8mm_vpu_sels)); + hws[IMX8MM_CLK_GPU3D_SRC] = imx_clk_hw_mux2("gpu3d_src", base + 0x8180, 24, 3, imx8mm_gpu3d_sels, ARRAY_SIZE(imx8mm_gpu3d_sels)); + hws[IMX8MM_CLK_GPU2D_SRC] = imx_clk_hw_mux2("gpu2d_src", base + 0x8200, 24, 3, imx8mm_gpu2d_sels, ARRAY_SIZE(imx8mm_gpu2d_sels)); + hws[IMX8MM_CLK_A53_CG] = imx_clk_hw_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28); + hws[IMX8MM_CLK_M4_CG] = imx_clk_hw_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); + hws[IMX8MM_CLK_VPU_CG] = imx_clk_hw_gate3("vpu_cg", "vpu_src", base + 0x8100, 28); + hws[IMX8MM_CLK_GPU3D_CG] = imx_clk_hw_gate3("gpu3d_cg", "gpu3d_src", base + 0x8180, 28); + hws[IMX8MM_CLK_GPU2D_CG] = imx_clk_hw_gate3("gpu2d_cg", "gpu2d_src", base + 0x8200, 28); + hws[IMX8MM_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); + hws[IMX8MM_CLK_M4_DIV] = imx_clk_hw_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); + hws[IMX8MM_CLK_VPU_DIV] = imx_clk_hw_divider2("vpu_div", "vpu_cg", base + 0x8100, 0, 3); + hws[IMX8MM_CLK_GPU3D_DIV] = imx_clk_hw_divider2("gpu3d_div", "gpu3d_cg", base + 0x8180, 0, 3); + hws[IMX8MM_CLK_GPU2D_DIV] = imx_clk_hw_divider2("gpu2d_div", "gpu2d_cg", base + 0x8200, 0, 3); /* BUS */ - clks[IMX8MM_CLK_MAIN_AXI] = imx8m_clk_composite_critical("main_axi", imx8mm_main_axi_sels, base + 0x8800); - clks[IMX8MM_CLK_ENET_AXI] = imx8m_clk_composite("enet_axi", imx8mm_enet_axi_sels, base + 0x8880); - clks[IMX8MM_CLK_NAND_USDHC_BUS] = imx8m_clk_composite_critical("nand_usdhc_bus", imx8mm_nand_usdhc_sels, base + 0x8900); - clks[IMX8MM_CLK_VPU_BUS] = imx8m_clk_composite("vpu_bus", imx8mm_vpu_bus_sels, base + 0x8980); - clks[IMX8MM_CLK_DISP_AXI] = imx8m_clk_composite("disp_axi", imx8mm_disp_axi_sels, base + 0x8a00); - clks[IMX8MM_CLK_DISP_APB] = imx8m_clk_composite("disp_apb", imx8mm_disp_apb_sels, base + 0x8a80); - clks[IMX8MM_CLK_DISP_RTRM] = imx8m_clk_composite("disp_rtrm", imx8mm_disp_rtrm_sels, base + 0x8b00); - clks[IMX8MM_CLK_USB_BUS] = imx8m_clk_composite("usb_bus", imx8mm_usb_bus_sels, base + 0x8b80); - clks[IMX8MM_CLK_GPU_AXI] = imx8m_clk_composite("gpu_axi", imx8mm_gpu_axi_sels, base + 0x8c00); - clks[IMX8MM_CLK_GPU_AHB] = imx8m_clk_composite("gpu_ahb", imx8mm_gpu_ahb_sels, base + 0x8c80); - clks[IMX8MM_CLK_NOC] = imx8m_clk_composite_critical("noc", imx8mm_noc_sels, base + 0x8d00); - clks[IMX8MM_CLK_NOC_APB] = imx8m_clk_composite_critical("noc_apb", imx8mm_noc_apb_sels, base + 0x8d80); + hws[IMX8MM_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi", imx8mm_main_axi_sels, base + 0x8800); + hws[IMX8MM_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mm_enet_axi_sels, base + 0x8880); + hws[IMX8MM_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite_critical("nand_usdhc_bus", imx8mm_nand_usdhc_sels, base + 0x8900); + hws[IMX8MM_CLK_VPU_BUS] = imx8m_clk_hw_composite("vpu_bus", imx8mm_vpu_bus_sels, base + 0x8980); + hws[IMX8MM_CLK_DISP_AXI] = imx8m_clk_hw_composite("disp_axi", imx8mm_disp_axi_sels, base + 0x8a00); + hws[IMX8MM_CLK_DISP_APB] = imx8m_clk_hw_composite("disp_apb", imx8mm_disp_apb_sels, base + 0x8a80); + hws[IMX8MM_CLK_DISP_RTRM] = imx8m_clk_hw_composite("disp_rtrm", imx8mm_disp_rtrm_sels, base + 0x8b00); + hws[IMX8MM_CLK_USB_BUS] = imx8m_clk_hw_composite("usb_bus", imx8mm_usb_bus_sels, base + 0x8b80); + hws[IMX8MM_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mm_gpu_axi_sels, base + 0x8c00); + hws[IMX8MM_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mm_gpu_ahb_sels, base + 0x8c80); + hws[IMX8MM_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mm_noc_sels, base + 0x8d00); + hws[IMX8MM_CLK_NOC_APB] = imx8m_clk_hw_composite_critical("noc_apb", imx8mm_noc_apb_sels, base + 0x8d80); /* AHB */ - clks[IMX8MM_CLK_AHB] = imx8m_clk_composite_critical("ahb", imx8mm_ahb_sels, base + 0x9000); - clks[IMX8MM_CLK_AUDIO_AHB] = imx8m_clk_composite("audio_ahb", imx8mm_audio_ahb_sels, base + 0x9100); + hws[IMX8MM_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb", imx8mm_ahb_sels, base + 0x9000); + hws[IMX8MM_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mm_audio_ahb_sels, base + 0x9100); /* IPG */ - clks[IMX8MM_CLK_IPG_ROOT] = imx_clk_divider2("ipg_root", "ahb", base + 0x9080, 0, 1); - clks[IMX8MM_CLK_IPG_AUDIO_ROOT] = imx_clk_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1); + hws[IMX8MM_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb", base + 0x9080, 0, 1); + hws[IMX8MM_CLK_IPG_AUDIO_ROOT] = imx_clk_hw_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1); + + /* + * DRAM clocks are manipulated from TF-A outside clock framework. + * Mark with GET_RATE_NOCACHE to always read div value from hardware + */ + hws[IMX8MM_CLK_DRAM_ALT] = __imx8m_clk_hw_composite("dram_alt", imx8mm_dram_alt_sels, base + 0xa000, CLK_GET_RATE_NOCACHE); + hws[IMX8MM_CLK_DRAM_APB] = __imx8m_clk_hw_composite("dram_apb", imx8mm_dram_apb_sels, base + 0xa080, CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); /* IP */ - clks[IMX8MM_CLK_DRAM_ALT] = imx8m_clk_composite("dram_alt", imx8mm_dram_alt_sels, base + 0xa000); - clks[IMX8MM_CLK_DRAM_APB] = imx8m_clk_composite_critical("dram_apb", imx8mm_dram_apb_sels, base + 0xa080); - clks[IMX8MM_CLK_VPU_G1] = imx8m_clk_composite("vpu_g1", imx8mm_vpu_g1_sels, base + 0xa100); - clks[IMX8MM_CLK_VPU_G2] = imx8m_clk_composite("vpu_g2", imx8mm_vpu_g2_sels, base + 0xa180); - clks[IMX8MM_CLK_DISP_DTRC] = imx8m_clk_composite("disp_dtrc", imx8mm_disp_dtrc_sels, base + 0xa200); - clks[IMX8MM_CLK_DISP_DC8000] = imx8m_clk_composite("disp_dc8000", imx8mm_disp_dc8000_sels, base + 0xa280); - clks[IMX8MM_CLK_PCIE1_CTRL] = imx8m_clk_composite("pcie1_ctrl", imx8mm_pcie1_ctrl_sels, base + 0xa300); - clks[IMX8MM_CLK_PCIE1_PHY] = imx8m_clk_composite("pcie1_phy", imx8mm_pcie1_phy_sels, base + 0xa380); - clks[IMX8MM_CLK_PCIE1_AUX] = imx8m_clk_composite("pcie1_aux", imx8mm_pcie1_aux_sels, base + 0xa400); - clks[IMX8MM_CLK_DC_PIXEL] = imx8m_clk_composite("dc_pixel", imx8mm_dc_pixel_sels, base + 0xa480); - clks[IMX8MM_CLK_LCDIF_PIXEL] = imx8m_clk_composite("lcdif_pixel", imx8mm_lcdif_pixel_sels, base + 0xa500); - clks[IMX8MM_CLK_SAI1] = imx8m_clk_composite("sai1", imx8mm_sai1_sels, base + 0xa580); - clks[IMX8MM_CLK_SAI2] = imx8m_clk_composite("sai2", imx8mm_sai2_sels, base + 0xa600); - clks[IMX8MM_CLK_SAI3] = imx8m_clk_composite("sai3", imx8mm_sai3_sels, base + 0xa680); - clks[IMX8MM_CLK_SAI4] = imx8m_clk_composite("sai4", imx8mm_sai4_sels, base + 0xa700); - clks[IMX8MM_CLK_SAI5] = imx8m_clk_composite("sai5", imx8mm_sai5_sels, base + 0xa780); - clks[IMX8MM_CLK_SAI6] = imx8m_clk_composite("sai6", imx8mm_sai6_sels, base + 0xa800); - clks[IMX8MM_CLK_SPDIF1] = imx8m_clk_composite("spdif1", imx8mm_spdif1_sels, base + 0xa880); - clks[IMX8MM_CLK_SPDIF2] = imx8m_clk_composite("spdif2", imx8mm_spdif2_sels, base + 0xa900); - clks[IMX8MM_CLK_ENET_REF] = imx8m_clk_composite("enet_ref", imx8mm_enet_ref_sels, base + 0xa980); - clks[IMX8MM_CLK_ENET_TIMER] = imx8m_clk_composite("enet_timer", imx8mm_enet_timer_sels, base + 0xaa00); - clks[IMX8MM_CLK_ENET_PHY_REF] = imx8m_clk_composite("enet_phy", imx8mm_enet_phy_sels, base + 0xaa80); - clks[IMX8MM_CLK_NAND] = imx8m_clk_composite("nand", imx8mm_nand_sels, base + 0xab00); - clks[IMX8MM_CLK_QSPI] = imx8m_clk_composite("qspi", imx8mm_qspi_sels, base + 0xab80); - clks[IMX8MM_CLK_USDHC1] = imx8m_clk_composite("usdhc1", imx8mm_usdhc1_sels, base + 0xac00); - clks[IMX8MM_CLK_USDHC2] = imx8m_clk_composite("usdhc2", imx8mm_usdhc2_sels, base + 0xac80); - clks[IMX8MM_CLK_I2C1] = imx8m_clk_composite("i2c1", imx8mm_i2c1_sels, base + 0xad00); - clks[IMX8MM_CLK_I2C2] = imx8m_clk_composite("i2c2", imx8mm_i2c2_sels, base + 0xad80); - clks[IMX8MM_CLK_I2C3] = imx8m_clk_composite("i2c3", imx8mm_i2c3_sels, base + 0xae00); - clks[IMX8MM_CLK_I2C4] = imx8m_clk_composite("i2c4", imx8mm_i2c4_sels, base + 0xae80); - clks[IMX8MM_CLK_UART1] = imx8m_clk_composite("uart1", imx8mm_uart1_sels, base + 0xaf00); - clks[IMX8MM_CLK_UART2] = imx8m_clk_composite("uart2", imx8mm_uart2_sels, base + 0xaf80); - clks[IMX8MM_CLK_UART3] = imx8m_clk_composite("uart3", imx8mm_uart3_sels, base + 0xb000); - clks[IMX8MM_CLK_UART4] = imx8m_clk_composite("uart4", imx8mm_uart4_sels, base + 0xb080); - clks[IMX8MM_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mm_usb_core_sels, base + 0xb100); - clks[IMX8MM_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mm_usb_phy_sels, base + 0xb180); - clks[IMX8MM_CLK_GIC] = imx8m_clk_composite_critical("gic", imx8mm_gic_sels, base + 0xb200); - clks[IMX8MM_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mm_ecspi1_sels, base + 0xb280); - clks[IMX8MM_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mm_ecspi2_sels, base + 0xb300); - clks[IMX8MM_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mm_pwm1_sels, base + 0xb380); - clks[IMX8MM_CLK_PWM2] = imx8m_clk_composite("pwm2", imx8mm_pwm2_sels, base + 0xb400); - clks[IMX8MM_CLK_PWM3] = imx8m_clk_composite("pwm3", imx8mm_pwm3_sels, base + 0xb480); - clks[IMX8MM_CLK_PWM4] = imx8m_clk_composite("pwm4", imx8mm_pwm4_sels, base + 0xb500); - clks[IMX8MM_CLK_GPT1] = imx8m_clk_composite("gpt1", imx8mm_gpt1_sels, base + 0xb580); - clks[IMX8MM_CLK_WDOG] = imx8m_clk_composite("wdog", imx8mm_wdog_sels, base + 0xb900); - clks[IMX8MM_CLK_WRCLK] = imx8m_clk_composite("wrclk", imx8mm_wrclk_sels, base + 0xb980); - clks[IMX8MM_CLK_CLKO1] = imx8m_clk_composite("clko1", imx8mm_clko1_sels, base + 0xba00); - clks[IMX8MM_CLK_DSI_CORE] = imx8m_clk_composite("dsi_core", imx8mm_dsi_core_sels, base + 0xbb00); - clks[IMX8MM_CLK_DSI_PHY_REF] = imx8m_clk_composite("dsi_phy_ref", imx8mm_dsi_phy_sels, base + 0xbb80); - clks[IMX8MM_CLK_DSI_DBI] = imx8m_clk_composite("dsi_dbi", imx8mm_dsi_dbi_sels, base + 0xbc00); - clks[IMX8MM_CLK_USDHC3] = imx8m_clk_composite("usdhc3", imx8mm_usdhc3_sels, base + 0xbc80); - clks[IMX8MM_CLK_CSI1_CORE] = imx8m_clk_composite("csi1_core", imx8mm_csi1_core_sels, base + 0xbd00); - clks[IMX8MM_CLK_CSI1_PHY_REF] = imx8m_clk_composite("csi1_phy_ref", imx8mm_csi1_phy_sels, base + 0xbd80); - clks[IMX8MM_CLK_CSI1_ESC] = imx8m_clk_composite("csi1_esc", imx8mm_csi1_esc_sels, base + 0xbe00); - clks[IMX8MM_CLK_CSI2_CORE] = imx8m_clk_composite("csi2_core", imx8mm_csi2_core_sels, base + 0xbe80); - clks[IMX8MM_CLK_CSI2_PHY_REF] = imx8m_clk_composite("csi2_phy_ref", imx8mm_csi2_phy_sels, base + 0xbf00); - clks[IMX8MM_CLK_CSI2_ESC] = imx8m_clk_composite("csi2_esc", imx8mm_csi2_esc_sels, base + 0xbf80); - clks[IMX8MM_CLK_PCIE2_CTRL] = imx8m_clk_composite("pcie2_ctrl", imx8mm_pcie2_ctrl_sels, base + 0xc000); - clks[IMX8MM_CLK_PCIE2_PHY] = imx8m_clk_composite("pcie2_phy", imx8mm_pcie2_phy_sels, base + 0xc080); - clks[IMX8MM_CLK_PCIE2_AUX] = imx8m_clk_composite("pcie2_aux", imx8mm_pcie2_aux_sels, base + 0xc100); - clks[IMX8MM_CLK_ECSPI3] = imx8m_clk_composite("ecspi3", imx8mm_ecspi3_sels, base + 0xc180); - clks[IMX8MM_CLK_PDM] = imx8m_clk_composite("pdm", imx8mm_pdm_sels, base + 0xc200); - clks[IMX8MM_CLK_VPU_H1] = imx8m_clk_composite("vpu_h1", imx8mm_vpu_h1_sels, base + 0xc280); + hws[IMX8MM_CLK_VPU_G1] = imx8m_clk_hw_composite("vpu_g1", imx8mm_vpu_g1_sels, base + 0xa100); + hws[IMX8MM_CLK_VPU_G2] = imx8m_clk_hw_composite("vpu_g2", imx8mm_vpu_g2_sels, base + 0xa180); + hws[IMX8MM_CLK_DISP_DTRC] = imx8m_clk_hw_composite("disp_dtrc", imx8mm_disp_dtrc_sels, base + 0xa200); + hws[IMX8MM_CLK_DISP_DC8000] = imx8m_clk_hw_composite("disp_dc8000", imx8mm_disp_dc8000_sels, base + 0xa280); + hws[IMX8MM_CLK_PCIE1_CTRL] = imx8m_clk_hw_composite("pcie1_ctrl", imx8mm_pcie1_ctrl_sels, base + 0xa300); + hws[IMX8MM_CLK_PCIE1_PHY] = imx8m_clk_hw_composite("pcie1_phy", imx8mm_pcie1_phy_sels, base + 0xa380); + hws[IMX8MM_CLK_PCIE1_AUX] = imx8m_clk_hw_composite("pcie1_aux", imx8mm_pcie1_aux_sels, base + 0xa400); + hws[IMX8MM_CLK_DC_PIXEL] = imx8m_clk_hw_composite("dc_pixel", imx8mm_dc_pixel_sels, base + 0xa480); + hws[IMX8MM_CLK_LCDIF_PIXEL] = imx8m_clk_hw_composite("lcdif_pixel", imx8mm_lcdif_pixel_sels, base + 0xa500); + hws[IMX8MM_CLK_SAI1] = imx8m_clk_hw_composite("sai1", imx8mm_sai1_sels, base + 0xa580); + hws[IMX8MM_CLK_SAI2] = imx8m_clk_hw_composite("sai2", imx8mm_sai2_sels, base + 0xa600); + hws[IMX8MM_CLK_SAI3] = imx8m_clk_hw_composite("sai3", imx8mm_sai3_sels, base + 0xa680); + hws[IMX8MM_CLK_SAI4] = imx8m_clk_hw_composite("sai4", imx8mm_sai4_sels, base + 0xa700); + hws[IMX8MM_CLK_SAI5] = imx8m_clk_hw_composite("sai5", imx8mm_sai5_sels, base + 0xa780); + hws[IMX8MM_CLK_SAI6] = imx8m_clk_hw_composite("sai6", imx8mm_sai6_sels, base + 0xa800); + hws[IMX8MM_CLK_SPDIF1] = imx8m_clk_hw_composite("spdif1", imx8mm_spdif1_sels, base + 0xa880); + hws[IMX8MM_CLK_SPDIF2] = imx8m_clk_hw_composite("spdif2", imx8mm_spdif2_sels, base + 0xa900); + hws[IMX8MM_CLK_ENET_REF] = imx8m_clk_hw_composite("enet_ref", imx8mm_enet_ref_sels, base + 0xa980); + hws[IMX8MM_CLK_ENET_TIMER] = imx8m_clk_hw_composite("enet_timer", imx8mm_enet_timer_sels, base + 0xaa00); + hws[IMX8MM_CLK_ENET_PHY_REF] = imx8m_clk_hw_composite("enet_phy", imx8mm_enet_phy_sels, base + 0xaa80); + hws[IMX8MM_CLK_NAND] = imx8m_clk_hw_composite("nand", imx8mm_nand_sels, base + 0xab00); + hws[IMX8MM_CLK_QSPI] = imx8m_clk_hw_composite("qspi", imx8mm_qspi_sels, base + 0xab80); + hws[IMX8MM_CLK_USDHC1] = imx8m_clk_hw_composite("usdhc1", imx8mm_usdhc1_sels, base + 0xac00); + hws[IMX8MM_CLK_USDHC2] = imx8m_clk_hw_composite("usdhc2", imx8mm_usdhc2_sels, base + 0xac80); + hws[IMX8MM_CLK_I2C1] = imx8m_clk_hw_composite("i2c1", imx8mm_i2c1_sels, base + 0xad00); + hws[IMX8MM_CLK_I2C2] = imx8m_clk_hw_composite("i2c2", imx8mm_i2c2_sels, base + 0xad80); + hws[IMX8MM_CLK_I2C3] = imx8m_clk_hw_composite("i2c3", imx8mm_i2c3_sels, base + 0xae00); + hws[IMX8MM_CLK_I2C4] = imx8m_clk_hw_composite("i2c4", imx8mm_i2c4_sels, base + 0xae80); + hws[IMX8MM_CLK_UART1] = imx8m_clk_hw_composite("uart1", imx8mm_uart1_sels, base + 0xaf00); + hws[IMX8MM_CLK_UART2] = imx8m_clk_hw_composite("uart2", imx8mm_uart2_sels, base + 0xaf80); + hws[IMX8MM_CLK_UART3] = imx8m_clk_hw_composite("uart3", imx8mm_uart3_sels, base + 0xb000); + hws[IMX8MM_CLK_UART4] = imx8m_clk_hw_composite("uart4", imx8mm_uart4_sels, base + 0xb080); + hws[IMX8MM_CLK_USB_CORE_REF] = imx8m_clk_hw_composite("usb_core_ref", imx8mm_usb_core_sels, base + 0xb100); + hws[IMX8MM_CLK_USB_PHY_REF] = imx8m_clk_hw_composite("usb_phy_ref", imx8mm_usb_phy_sels, base + 0xb180); + hws[IMX8MM_CLK_GIC] = imx8m_clk_hw_composite_critical("gic", imx8mm_gic_sels, base + 0xb200); + hws[IMX8MM_CLK_ECSPI1] = imx8m_clk_hw_composite("ecspi1", imx8mm_ecspi1_sels, base + 0xb280); + hws[IMX8MM_CLK_ECSPI2] = imx8m_clk_hw_composite("ecspi2", imx8mm_ecspi2_sels, base + 0xb300); + hws[IMX8MM_CLK_PWM1] = imx8m_clk_hw_composite("pwm1", imx8mm_pwm1_sels, base + 0xb380); + hws[IMX8MM_CLK_PWM2] = imx8m_clk_hw_composite("pwm2", imx8mm_pwm2_sels, base + 0xb400); + hws[IMX8MM_CLK_PWM3] = imx8m_clk_hw_composite("pwm3", imx8mm_pwm3_sels, base + 0xb480); + hws[IMX8MM_CLK_PWM4] = imx8m_clk_hw_composite("pwm4", imx8mm_pwm4_sels, base + 0xb500); + hws[IMX8MM_CLK_GPT1] = imx8m_clk_hw_composite("gpt1", imx8mm_gpt1_sels, base + 0xb580); + hws[IMX8MM_CLK_WDOG] = imx8m_clk_hw_composite("wdog", imx8mm_wdog_sels, base + 0xb900); + hws[IMX8MM_CLK_WRCLK] = imx8m_clk_hw_composite("wrclk", imx8mm_wrclk_sels, base + 0xb980); + hws[IMX8MM_CLK_CLKO1] = imx8m_clk_hw_composite("clko1", imx8mm_clko1_sels, base + 0xba00); + hws[IMX8MM_CLK_DSI_CORE] = imx8m_clk_hw_composite("dsi_core", imx8mm_dsi_core_sels, base + 0xbb00); + hws[IMX8MM_CLK_DSI_PHY_REF] = imx8m_clk_hw_composite("dsi_phy_ref", imx8mm_dsi_phy_sels, base + 0xbb80); + hws[IMX8MM_CLK_DSI_DBI] = imx8m_clk_hw_composite("dsi_dbi", imx8mm_dsi_dbi_sels, base + 0xbc00); + hws[IMX8MM_CLK_USDHC3] = imx8m_clk_hw_composite("usdhc3", imx8mm_usdhc3_sels, base + 0xbc80); + hws[IMX8MM_CLK_CSI1_CORE] = imx8m_clk_hw_composite("csi1_core", imx8mm_csi1_core_sels, base + 0xbd00); + hws[IMX8MM_CLK_CSI1_PHY_REF] = imx8m_clk_hw_composite("csi1_phy_ref", imx8mm_csi1_phy_sels, base + 0xbd80); + hws[IMX8MM_CLK_CSI1_ESC] = imx8m_clk_hw_composite("csi1_esc", imx8mm_csi1_esc_sels, base + 0xbe00); + hws[IMX8MM_CLK_CSI2_CORE] = imx8m_clk_hw_composite("csi2_core", imx8mm_csi2_core_sels, base + 0xbe80); + hws[IMX8MM_CLK_CSI2_PHY_REF] = imx8m_clk_hw_composite("csi2_phy_ref", imx8mm_csi2_phy_sels, base + 0xbf00); + hws[IMX8MM_CLK_CSI2_ESC] = imx8m_clk_hw_composite("csi2_esc", imx8mm_csi2_esc_sels, base + 0xbf80); + hws[IMX8MM_CLK_PCIE2_CTRL] = imx8m_clk_hw_composite("pcie2_ctrl", imx8mm_pcie2_ctrl_sels, base + 0xc000); + hws[IMX8MM_CLK_PCIE2_PHY] = imx8m_clk_hw_composite("pcie2_phy", imx8mm_pcie2_phy_sels, base + 0xc080); + hws[IMX8MM_CLK_PCIE2_AUX] = imx8m_clk_hw_composite("pcie2_aux", imx8mm_pcie2_aux_sels, base + 0xc100); + hws[IMX8MM_CLK_ECSPI3] = imx8m_clk_hw_composite("ecspi3", imx8mm_ecspi3_sels, base + 0xc180); + hws[IMX8MM_CLK_PDM] = imx8m_clk_hw_composite("pdm", imx8mm_pdm_sels, base + 0xc200); + hws[IMX8MM_CLK_VPU_H1] = imx8m_clk_hw_composite("vpu_h1", imx8mm_vpu_h1_sels, base + 0xc280); /* CCGR */ - clks[IMX8MM_CLK_ECSPI1_ROOT] = imx_clk_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0); - clks[IMX8MM_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); - clks[IMX8MM_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); - clks[IMX8MM_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); - clks[IMX8MM_CLK_GPIO1_ROOT] = imx_clk_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); - clks[IMX8MM_CLK_GPIO2_ROOT] = imx_clk_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); - clks[IMX8MM_CLK_GPIO3_ROOT] = imx_clk_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); - clks[IMX8MM_CLK_GPIO4_ROOT] = imx_clk_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); - clks[IMX8MM_CLK_GPIO5_ROOT] = imx_clk_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); - clks[IMX8MM_CLK_GPT1_ROOT] = imx_clk_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0); - clks[IMX8MM_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); - clks[IMX8MM_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); - clks[IMX8MM_CLK_I2C3_ROOT] = imx_clk_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0); - clks[IMX8MM_CLK_I2C4_ROOT] = imx_clk_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0); - clks[IMX8MM_CLK_MU_ROOT] = imx_clk_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0); - clks[IMX8MM_CLK_OCOTP_ROOT] = imx_clk_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0); - clks[IMX8MM_CLK_PCIE1_ROOT] = imx_clk_gate4("pcie1_root_clk", "pcie1_ctrl", base + 0x4250, 0); - clks[IMX8MM_CLK_PWM1_ROOT] = imx_clk_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0); - clks[IMX8MM_CLK_PWM2_ROOT] = imx_clk_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0); - clks[IMX8MM_CLK_PWM3_ROOT] = imx_clk_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0); - clks[IMX8MM_CLK_PWM4_ROOT] = imx_clk_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0); - clks[IMX8MM_CLK_QSPI_ROOT] = imx_clk_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0); - clks[IMX8MM_CLK_NAND_ROOT] = imx_clk_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand); - clks[IMX8MM_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand); - clks[IMX8MM_CLK_SAI1_ROOT] = imx_clk_gate2_shared2("sai1_root_clk", "sai1", base + 0x4330, 0, &share_count_sai1); - clks[IMX8MM_CLK_SAI1_IPG] = imx_clk_gate2_shared2("sai1_ipg_clk", "ipg_audio_root", base + 0x4330, 0, &share_count_sai1); - clks[IMX8MM_CLK_SAI2_ROOT] = imx_clk_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2); - clks[IMX8MM_CLK_SAI2_IPG] = imx_clk_gate2_shared2("sai2_ipg_clk", "ipg_audio_root", base + 0x4340, 0, &share_count_sai2); - clks[IMX8MM_CLK_SAI3_ROOT] = imx_clk_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3); - clks[IMX8MM_CLK_SAI3_IPG] = imx_clk_gate2_shared2("sai3_ipg_clk", "ipg_audio_root", base + 0x4350, 0, &share_count_sai3); - clks[IMX8MM_CLK_SAI4_ROOT] = imx_clk_gate2_shared2("sai4_root_clk", "sai4", base + 0x4360, 0, &share_count_sai4); - clks[IMX8MM_CLK_SAI4_IPG] = imx_clk_gate2_shared2("sai4_ipg_clk", "ipg_audio_root", base + 0x4360, 0, &share_count_sai4); - clks[IMX8MM_CLK_SAI5_ROOT] = imx_clk_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5); - clks[IMX8MM_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); - clks[IMX8MM_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); - clks[IMX8MM_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); - clks[IMX8MM_CLK_SNVS_ROOT] = imx_clk_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0); - clks[IMX8MM_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); - clks[IMX8MM_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); - clks[IMX8MM_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); - clks[IMX8MM_CLK_UART4_ROOT] = imx_clk_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0); - clks[IMX8MM_CLK_USB1_CTRL_ROOT] = imx_clk_gate4("usb1_ctrl_root_clk", "usb_bus", base + 0x44d0, 0); - clks[IMX8MM_CLK_GPU3D_ROOT] = imx_clk_gate4("gpu3d_root_clk", "gpu3d_div", base + 0x44f0, 0); - clks[IMX8MM_CLK_USDHC1_ROOT] = imx_clk_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0); - clks[IMX8MM_CLK_USDHC2_ROOT] = imx_clk_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0); - clks[IMX8MM_CLK_WDOG1_ROOT] = imx_clk_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0); - clks[IMX8MM_CLK_WDOG2_ROOT] = imx_clk_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0); - clks[IMX8MM_CLK_WDOG3_ROOT] = imx_clk_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0); - clks[IMX8MM_CLK_VPU_G1_ROOT] = imx_clk_gate4("vpu_g1_root_clk", "vpu_g1", base + 0x4560, 0); - clks[IMX8MM_CLK_GPU_BUS_ROOT] = imx_clk_gate4("gpu_root_clk", "gpu_axi", base + 0x4570, 0); - clks[IMX8MM_CLK_VPU_H1_ROOT] = imx_clk_gate4("vpu_h1_root_clk", "vpu_h1", base + 0x4590, 0); - clks[IMX8MM_CLK_VPU_G2_ROOT] = imx_clk_gate4("vpu_g2_root_clk", "vpu_g2", base + 0x45a0, 0); - clks[IMX8MM_CLK_PDM_ROOT] = imx_clk_gate2_shared2("pdm_root_clk", "pdm", base + 0x45b0, 0, &share_count_pdm); - clks[IMX8MM_CLK_PDM_IPG] = imx_clk_gate2_shared2("pdm_ipg_clk", "ipg_audio_root", base + 0x45b0, 0, &share_count_pdm); - clks[IMX8MM_CLK_DISP_ROOT] = imx_clk_gate2_shared2("disp_root_clk", "disp_dc8000", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MM_CLK_DISP_AXI_ROOT] = imx_clk_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MM_CLK_DISP_APB_ROOT] = imx_clk_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MM_CLK_DISP_RTRM_ROOT] = imx_clk_gate2_shared2("disp_rtrm_root_clk", "disp_rtrm", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MM_CLK_USDHC3_ROOT] = imx_clk_gate4("usdhc3_root_clk", "usdhc3", base + 0x45e0, 0); - clks[IMX8MM_CLK_TMU_ROOT] = imx_clk_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0); - clks[IMX8MM_CLK_VPU_DEC_ROOT] = imx_clk_gate4("vpu_dec_root_clk", "vpu_bus", base + 0x4630, 0); - clks[IMX8MM_CLK_SDMA1_ROOT] = imx_clk_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0); - clks[IMX8MM_CLK_SDMA2_ROOT] = imx_clk_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0); - clks[IMX8MM_CLK_SDMA3_ROOT] = imx_clk_gate4("sdma3_clk", "ipg_audio_root", base + 0x45f0, 0); - clks[IMX8MM_CLK_GPU2D_ROOT] = imx_clk_gate4("gpu2d_root_clk", "gpu2d_div", base + 0x4660, 0); - clks[IMX8MM_CLK_CSI1_ROOT] = imx_clk_gate4("csi1_root_clk", "csi1_core", base + 0x4650, 0); - - clks[IMX8MM_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc_24m", 1, 8); - - clks[IMX8MM_CLK_DRAM_ALT_ROOT] = imx_clk_fixed_factor("dram_alt_root", "dram_alt", 1, 4); - clks[IMX8MM_CLK_DRAM_CORE] = imx_clk_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mm_dram_core_sels, ARRAY_SIZE(imx8mm_dram_core_sels), CLK_IS_CRITICAL); - - clks[IMX8MM_CLK_ARM] = imx_clk_cpu("arm", "arm_a53_div", - clks[IMX8MM_CLK_A53_DIV], - clks[IMX8MM_CLK_A53_SRC], - clks[IMX8MM_ARM_PLL_OUT], - clks[IMX8MM_SYS_PLL1_800M]); - - imx_check_clocks(clks, ARRAY_SIZE(clks)); - - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + hws[IMX8MM_CLK_ECSPI1_ROOT] = imx_clk_hw_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0); + hws[IMX8MM_CLK_ECSPI2_ROOT] = imx_clk_hw_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); + hws[IMX8MM_CLK_ECSPI3_ROOT] = imx_clk_hw_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); + hws[IMX8MM_CLK_ENET1_ROOT] = imx_clk_hw_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); + hws[IMX8MM_CLK_GPIO1_ROOT] = imx_clk_hw_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); + hws[IMX8MM_CLK_GPIO2_ROOT] = imx_clk_hw_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); + hws[IMX8MM_CLK_GPIO3_ROOT] = imx_clk_hw_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); + hws[IMX8MM_CLK_GPIO4_ROOT] = imx_clk_hw_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); + hws[IMX8MM_CLK_GPIO5_ROOT] = imx_clk_hw_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); + hws[IMX8MM_CLK_GPT1_ROOT] = imx_clk_hw_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0); + hws[IMX8MM_CLK_I2C1_ROOT] = imx_clk_hw_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); + hws[IMX8MM_CLK_I2C2_ROOT] = imx_clk_hw_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); + hws[IMX8MM_CLK_I2C3_ROOT] = imx_clk_hw_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0); + hws[IMX8MM_CLK_I2C4_ROOT] = imx_clk_hw_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0); + hws[IMX8MM_CLK_MU_ROOT] = imx_clk_hw_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0); + hws[IMX8MM_CLK_OCOTP_ROOT] = imx_clk_hw_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0); + hws[IMX8MM_CLK_PCIE1_ROOT] = imx_clk_hw_gate4("pcie1_root_clk", "pcie1_ctrl", base + 0x4250, 0); + hws[IMX8MM_CLK_PWM1_ROOT] = imx_clk_hw_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0); + hws[IMX8MM_CLK_PWM2_ROOT] = imx_clk_hw_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0); + hws[IMX8MM_CLK_PWM3_ROOT] = imx_clk_hw_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0); + hws[IMX8MM_CLK_PWM4_ROOT] = imx_clk_hw_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0); + hws[IMX8MM_CLK_QSPI_ROOT] = imx_clk_hw_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0); + hws[IMX8MM_CLK_NAND_ROOT] = imx_clk_hw_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand); + hws[IMX8MM_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand); + hws[IMX8MM_CLK_SAI1_ROOT] = imx_clk_hw_gate2_shared2("sai1_root_clk", "sai1", base + 0x4330, 0, &share_count_sai1); + hws[IMX8MM_CLK_SAI1_IPG] = imx_clk_hw_gate2_shared2("sai1_ipg_clk", "ipg_audio_root", base + 0x4330, 0, &share_count_sai1); + hws[IMX8MM_CLK_SAI2_ROOT] = imx_clk_hw_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2); + hws[IMX8MM_CLK_SAI2_IPG] = imx_clk_hw_gate2_shared2("sai2_ipg_clk", "ipg_audio_root", base + 0x4340, 0, &share_count_sai2); + hws[IMX8MM_CLK_SAI3_ROOT] = imx_clk_hw_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3); + hws[IMX8MM_CLK_SAI3_IPG] = imx_clk_hw_gate2_shared2("sai3_ipg_clk", "ipg_audio_root", base + 0x4350, 0, &share_count_sai3); + hws[IMX8MM_CLK_SAI4_ROOT] = imx_clk_hw_gate2_shared2("sai4_root_clk", "sai4", base + 0x4360, 0, &share_count_sai4); + hws[IMX8MM_CLK_SAI4_IPG] = imx_clk_hw_gate2_shared2("sai4_ipg_clk", "ipg_audio_root", base + 0x4360, 0, &share_count_sai4); + hws[IMX8MM_CLK_SAI5_ROOT] = imx_clk_hw_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5); + hws[IMX8MM_CLK_SAI5_IPG] = imx_clk_hw_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); + hws[IMX8MM_CLK_SAI6_ROOT] = imx_clk_hw_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); + hws[IMX8MM_CLK_SAI6_IPG] = imx_clk_hw_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); + hws[IMX8MM_CLK_SNVS_ROOT] = imx_clk_hw_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0); + hws[IMX8MM_CLK_UART1_ROOT] = imx_clk_hw_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); + hws[IMX8MM_CLK_UART2_ROOT] = imx_clk_hw_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); + hws[IMX8MM_CLK_UART3_ROOT] = imx_clk_hw_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); + hws[IMX8MM_CLK_UART4_ROOT] = imx_clk_hw_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0); + hws[IMX8MM_CLK_USB1_CTRL_ROOT] = imx_clk_hw_gate4("usb1_ctrl_root_clk", "usb_bus", base + 0x44d0, 0); + hws[IMX8MM_CLK_GPU3D_ROOT] = imx_clk_hw_gate4("gpu3d_root_clk", "gpu3d_div", base + 0x44f0, 0); + hws[IMX8MM_CLK_USDHC1_ROOT] = imx_clk_hw_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0); + hws[IMX8MM_CLK_USDHC2_ROOT] = imx_clk_hw_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0); + hws[IMX8MM_CLK_WDOG1_ROOT] = imx_clk_hw_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0); + hws[IMX8MM_CLK_WDOG2_ROOT] = imx_clk_hw_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0); + hws[IMX8MM_CLK_WDOG3_ROOT] = imx_clk_hw_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0); + hws[IMX8MM_CLK_VPU_G1_ROOT] = imx_clk_hw_gate4("vpu_g1_root_clk", "vpu_g1", base + 0x4560, 0); + hws[IMX8MM_CLK_GPU_BUS_ROOT] = imx_clk_hw_gate4("gpu_root_clk", "gpu_axi", base + 0x4570, 0); + hws[IMX8MM_CLK_VPU_H1_ROOT] = imx_clk_hw_gate4("vpu_h1_root_clk", "vpu_h1", base + 0x4590, 0); + hws[IMX8MM_CLK_VPU_G2_ROOT] = imx_clk_hw_gate4("vpu_g2_root_clk", "vpu_g2", base + 0x45a0, 0); + hws[IMX8MM_CLK_PDM_ROOT] = imx_clk_hw_gate2_shared2("pdm_root_clk", "pdm", base + 0x45b0, 0, &share_count_pdm); + hws[IMX8MM_CLK_PDM_IPG] = imx_clk_hw_gate2_shared2("pdm_ipg_clk", "ipg_audio_root", base + 0x45b0, 0, &share_count_pdm); + hws[IMX8MM_CLK_DISP_ROOT] = imx_clk_hw_gate2_shared2("disp_root_clk", "disp_dc8000", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MM_CLK_DISP_AXI_ROOT] = imx_clk_hw_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MM_CLK_DISP_APB_ROOT] = imx_clk_hw_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MM_CLK_DISP_RTRM_ROOT] = imx_clk_hw_gate2_shared2("disp_rtrm_root_clk", "disp_rtrm", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MM_CLK_USDHC3_ROOT] = imx_clk_hw_gate4("usdhc3_root_clk", "usdhc3", base + 0x45e0, 0); + hws[IMX8MM_CLK_TMU_ROOT] = imx_clk_hw_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0); + hws[IMX8MM_CLK_VPU_DEC_ROOT] = imx_clk_hw_gate4("vpu_dec_root_clk", "vpu_bus", base + 0x4630, 0); + hws[IMX8MM_CLK_SDMA1_ROOT] = imx_clk_hw_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0); + hws[IMX8MM_CLK_SDMA2_ROOT] = imx_clk_hw_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0); + hws[IMX8MM_CLK_SDMA3_ROOT] = imx_clk_hw_gate4("sdma3_clk", "ipg_audio_root", base + 0x45f0, 0); + hws[IMX8MM_CLK_GPU2D_ROOT] = imx_clk_hw_gate4("gpu2d_root_clk", "gpu2d_div", base + 0x4660, 0); + hws[IMX8MM_CLK_CSI1_ROOT] = imx_clk_hw_gate4("csi1_root_clk", "csi1_core", base + 0x4650, 0); + + hws[IMX8MM_CLK_GPT_3M] = imx_clk_hw_fixed_factor("gpt_3m", "osc_24m", 1, 8); + + hws[IMX8MM_CLK_DRAM_ALT_ROOT] = imx_clk_hw_fixed_factor("dram_alt_root", "dram_alt", 1, 4); + hws[IMX8MM_CLK_DRAM_CORE] = imx_clk_hw_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mm_dram_core_sels, ARRAY_SIZE(imx8mm_dram_core_sels), CLK_IS_CRITICAL); + + hws[IMX8MM_CLK_ARM] = imx_clk_hw_cpu("arm", "arm_a53_div", + hws[IMX8MM_CLK_A53_DIV]->clk, + hws[IMX8MM_CLK_A53_SRC]->clk, + hws[IMX8MM_ARM_PLL_OUT]->clk, + hws[IMX8MM_SYS_PLL1_800M]->clk); + + imx_check_clk_hws(hws, IMX8MM_CLK_END); + + ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); if (ret < 0) { - pr_err("failed to register clks for i.MX8MM\n"); - goto unregister_clks; + dev_err(dev, "failed to register clks for i.MX8MM\n"); + goto unregister_hws; + } + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_hws[i] = &hws[index]->clk; } - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(uart_hws); return 0; -unregister_clks: - imx_unregister_clocks(clks, ARRAY_SIZE(clks)); +unregister_hws: + imx_unregister_hw_clocks(hws, IMX8MM_CLK_END); return ret; } @@ -616,6 +634,11 @@ static struct platform_driver imx8mm_clk_driver = { .probe = imx8mm_clocks_probe, .driver = { .name = "imx8mm-ccm", + /* + * Disable bind attributes: clocks are not removed and + * reloading the driver will crash or break devices. + */ + .suppress_bind_attrs = true, .of_match_table = of_match_ptr(imx8mm_clk_of_match), }, }; diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c index 9f5a5a56b45e..c5e7316b4c66 100644 --- a/drivers/clk/imx/clk-imx8mn.c +++ b/drivers/clk/imx/clk-imx8mn.c @@ -12,6 +12,7 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/platform_device.h> +#include <linux/slab.h> #include <linux/types.h> #include "clk.h" @@ -280,284 +281,302 @@ static const char * const imx8mn_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sy "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", "osc_32k", }; -static struct clk *clks[IMX8MN_CLK_END]; -static struct clk_onecell_data clk_data; +static struct clk_hw_onecell_data *clk_hw_data; +static struct clk_hw **hws; -static struct clk ** const uart_clks[] = { - &clks[IMX8MN_CLK_UART1_ROOT], - &clks[IMX8MN_CLK_UART2_ROOT], - &clks[IMX8MN_CLK_UART3_ROOT], - &clks[IMX8MN_CLK_UART4_ROOT], - NULL +static const int uart_clk_ids[] = { + IMX8MN_CLK_UART1_ROOT, + IMX8MN_CLK_UART2_ROOT, + IMX8MN_CLK_UART3_ROOT, + IMX8MN_CLK_UART4_ROOT, }; +static struct clk **uart_hws[ARRAY_SIZE(uart_clk_ids) + 1]; static int imx8mn_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; void __iomem *base; - int ret; + int ret, i; - clks[IMX8MN_CLK_DUMMY] = imx_clk_fixed("dummy", 0); - clks[IMX8MN_CLK_24M] = of_clk_get_by_name(np, "osc_24m"); - clks[IMX8MN_CLK_32K] = of_clk_get_by_name(np, "osc_32k"); - clks[IMX8MN_CLK_EXT1] = of_clk_get_by_name(np, "clk_ext1"); - clks[IMX8MN_CLK_EXT2] = of_clk_get_by_name(np, "clk_ext2"); - clks[IMX8MN_CLK_EXT3] = of_clk_get_by_name(np, "clk_ext3"); - clks[IMX8MN_CLK_EXT4] = of_clk_get_by_name(np, "clk_ext4"); + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX8MN_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return -ENOMEM; + + clk_hw_data->num = IMX8MN_CLK_END; + hws = clk_hw_data->hws; + + hws[IMX8MN_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); + hws[IMX8MN_CLK_24M] = imx_obtain_fixed_clk_hw(np, "osc_24m"); + hws[IMX8MN_CLK_32K] = imx_obtain_fixed_clk_hw(np, "osc_32k"); + hws[IMX8MN_CLK_EXT1] = imx_obtain_fixed_clk_hw(np, "clk_ext1"); + hws[IMX8MN_CLK_EXT2] = imx_obtain_fixed_clk_hw(np, "clk_ext2"); + hws[IMX8MN_CLK_EXT3] = imx_obtain_fixed_clk_hw(np, "clk_ext3"); + hws[IMX8MN_CLK_EXT4] = imx_obtain_fixed_clk_hw(np, "clk_ext4"); np = of_find_compatible_node(NULL, NULL, "fsl,imx8mn-anatop"); base = of_iomap(np, 0); if (WARN_ON(!base)) { ret = -ENOMEM; - goto unregister_clks; + goto unregister_hws; } - clks[IMX8MN_AUDIO_PLL1_REF_SEL] = imx_clk_mux("audio_pll1_ref_sel", base + 0x0, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MN_AUDIO_PLL2_REF_SEL] = imx_clk_mux("audio_pll2_ref_sel", base + 0x14, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MN_VIDEO_PLL1_REF_SEL] = imx_clk_mux("video_pll1_ref_sel", base + 0x28, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MN_DRAM_PLL_REF_SEL] = imx_clk_mux("dram_pll_ref_sel", base + 0x50, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MN_GPU_PLL_REF_SEL] = imx_clk_mux("gpu_pll_ref_sel", base + 0x64, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MN_VPU_PLL_REF_SEL] = imx_clk_mux("vpu_pll_ref_sel", base + 0x74, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MN_ARM_PLL_REF_SEL] = imx_clk_mux("arm_pll_ref_sel", base + 0x84, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MN_SYS_PLL3_REF_SEL] = imx_clk_mux("sys_pll3_ref_sel", base + 0x114, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - - clks[IMX8MN_AUDIO_PLL1] = imx_clk_pll14xx("audio_pll1", "audio_pll1_ref_sel", base, &imx_1443x_pll); - clks[IMX8MN_AUDIO_PLL2] = imx_clk_pll14xx("audio_pll2", "audio_pll2_ref_sel", base + 0x14, &imx_1443x_pll); - clks[IMX8MN_VIDEO_PLL1] = imx_clk_pll14xx("video_pll1", "video_pll1_ref_sel", base + 0x28, &imx_1443x_pll); - clks[IMX8MN_DRAM_PLL] = imx_clk_pll14xx("dram_pll", "dram_pll_ref_sel", base + 0x50, &imx_1443x_pll); - clks[IMX8MN_GPU_PLL] = imx_clk_pll14xx("gpu_pll", "gpu_pll_ref_sel", base + 0x64, &imx_1416x_pll); - clks[IMX8MN_VPU_PLL] = imx_clk_pll14xx("vpu_pll", "vpu_pll_ref_sel", base + 0x74, &imx_1416x_pll); - clks[IMX8MN_ARM_PLL] = imx_clk_pll14xx("arm_pll", "arm_pll_ref_sel", base + 0x84, &imx_1416x_pll); - clks[IMX8MN_SYS_PLL1] = imx_clk_fixed("sys_pll1", 800000000); - clks[IMX8MN_SYS_PLL2] = imx_clk_fixed("sys_pll2", 1000000000); - clks[IMX8MN_SYS_PLL3] = imx_clk_pll14xx("sys_pll3", "sys_pll3_ref_sel", base + 0x114, &imx_1416x_pll); + hws[IMX8MN_AUDIO_PLL1_REF_SEL] = imx_clk_hw_mux("audio_pll1_ref_sel", base + 0x0, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MN_AUDIO_PLL2_REF_SEL] = imx_clk_hw_mux("audio_pll2_ref_sel", base + 0x14, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MN_VIDEO_PLL1_REF_SEL] = imx_clk_hw_mux("video_pll1_ref_sel", base + 0x28, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MN_DRAM_PLL_REF_SEL] = imx_clk_hw_mux("dram_pll_ref_sel", base + 0x50, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MN_GPU_PLL_REF_SEL] = imx_clk_hw_mux("gpu_pll_ref_sel", base + 0x64, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MN_VPU_PLL_REF_SEL] = imx_clk_hw_mux("vpu_pll_ref_sel", base + 0x74, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MN_ARM_PLL_REF_SEL] = imx_clk_hw_mux("arm_pll_ref_sel", base + 0x84, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MN_SYS_PLL3_REF_SEL] = imx_clk_hw_mux("sys_pll3_ref_sel", base + 0x114, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + + hws[IMX8MN_AUDIO_PLL1] = imx_clk_hw_pll14xx("audio_pll1", "audio_pll1_ref_sel", base, &imx_1443x_pll); + hws[IMX8MN_AUDIO_PLL2] = imx_clk_hw_pll14xx("audio_pll2", "audio_pll2_ref_sel", base + 0x14, &imx_1443x_pll); + hws[IMX8MN_VIDEO_PLL1] = imx_clk_hw_pll14xx("video_pll1", "video_pll1_ref_sel", base + 0x28, &imx_1443x_pll); + hws[IMX8MN_DRAM_PLL] = imx_clk_hw_pll14xx("dram_pll", "dram_pll_ref_sel", base + 0x50, &imx_1443x_dram_pll); + hws[IMX8MN_GPU_PLL] = imx_clk_hw_pll14xx("gpu_pll", "gpu_pll_ref_sel", base + 0x64, &imx_1416x_pll); + hws[IMX8MN_VPU_PLL] = imx_clk_hw_pll14xx("vpu_pll", "vpu_pll_ref_sel", base + 0x74, &imx_1416x_pll); + hws[IMX8MN_ARM_PLL] = imx_clk_hw_pll14xx("arm_pll", "arm_pll_ref_sel", base + 0x84, &imx_1416x_pll); + hws[IMX8MN_SYS_PLL1] = imx_clk_hw_fixed("sys_pll1", 800000000); + hws[IMX8MN_SYS_PLL2] = imx_clk_hw_fixed("sys_pll2", 1000000000); + hws[IMX8MN_SYS_PLL3] = imx_clk_hw_pll14xx("sys_pll3", "sys_pll3_ref_sel", base + 0x114, &imx_1416x_pll); /* PLL bypass out */ - clks[IMX8MN_AUDIO_PLL1_BYPASS] = imx_clk_mux_flags("audio_pll1_bypass", base, 16, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MN_AUDIO_PLL2_BYPASS] = imx_clk_mux_flags("audio_pll2_bypass", base + 0x14, 16, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MN_VIDEO_PLL1_BYPASS] = imx_clk_mux_flags("video_pll1_bypass", base + 0x28, 16, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MN_DRAM_PLL_BYPASS] = imx_clk_mux_flags("dram_pll_bypass", base + 0x50, 16, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MN_GPU_PLL_BYPASS] = imx_clk_mux_flags("gpu_pll_bypass", base + 0x64, 28, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MN_VPU_PLL_BYPASS] = imx_clk_mux_flags("vpu_pll_bypass", base + 0x74, 28, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MN_ARM_PLL_BYPASS] = imx_clk_mux_flags("arm_pll_bypass", base + 0x84, 28, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MN_SYS_PLL3_BYPASS] = imx_clk_mux_flags("sys_pll3_bypass", base + 0x114, 28, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_AUDIO_PLL1_BYPASS] = imx_clk_hw_mux_flags("audio_pll1_bypass", base, 16, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_AUDIO_PLL2_BYPASS] = imx_clk_hw_mux_flags("audio_pll2_bypass", base + 0x14, 16, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_VIDEO_PLL1_BYPASS] = imx_clk_hw_mux_flags("video_pll1_bypass", base + 0x28, 16, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_DRAM_PLL_BYPASS] = imx_clk_hw_mux_flags("dram_pll_bypass", base + 0x50, 16, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_GPU_PLL_BYPASS] = imx_clk_hw_mux_flags("gpu_pll_bypass", base + 0x64, 28, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_VPU_PLL_BYPASS] = imx_clk_hw_mux_flags("vpu_pll_bypass", base + 0x74, 28, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_ARM_PLL_BYPASS] = imx_clk_hw_mux_flags("arm_pll_bypass", base + 0x84, 28, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MN_SYS_PLL3_BYPASS] = imx_clk_hw_mux_flags("sys_pll3_bypass", base + 0x114, 28, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT); /* PLL out gate */ - clks[IMX8MN_AUDIO_PLL1_OUT] = imx_clk_gate("audio_pll1_out", "audio_pll1_bypass", base, 13); - clks[IMX8MN_AUDIO_PLL2_OUT] = imx_clk_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x14, 13); - clks[IMX8MN_VIDEO_PLL1_OUT] = imx_clk_gate("video_pll1_out", "video_pll1_bypass", base + 0x28, 13); - clks[IMX8MN_DRAM_PLL_OUT] = imx_clk_gate("dram_pll_out", "dram_pll_bypass", base + 0x50, 13); - clks[IMX8MN_GPU_PLL_OUT] = imx_clk_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x64, 11); - clks[IMX8MN_VPU_PLL_OUT] = imx_clk_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x74, 11); - clks[IMX8MN_ARM_PLL_OUT] = imx_clk_gate("arm_pll_out", "arm_pll_bypass", base + 0x84, 11); - clks[IMX8MN_SYS_PLL3_OUT] = imx_clk_gate("sys_pll3_out", "sys_pll3_bypass", base + 0x114, 11); + hws[IMX8MN_AUDIO_PLL1_OUT] = imx_clk_hw_gate("audio_pll1_out", "audio_pll1_bypass", base, 13); + hws[IMX8MN_AUDIO_PLL2_OUT] = imx_clk_hw_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x14, 13); + hws[IMX8MN_VIDEO_PLL1_OUT] = imx_clk_hw_gate("video_pll1_out", "video_pll1_bypass", base + 0x28, 13); + hws[IMX8MN_DRAM_PLL_OUT] = imx_clk_hw_gate("dram_pll_out", "dram_pll_bypass", base + 0x50, 13); + hws[IMX8MN_GPU_PLL_OUT] = imx_clk_hw_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x64, 11); + hws[IMX8MN_VPU_PLL_OUT] = imx_clk_hw_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x74, 11); + hws[IMX8MN_ARM_PLL_OUT] = imx_clk_hw_gate("arm_pll_out", "arm_pll_bypass", base + 0x84, 11); + hws[IMX8MN_SYS_PLL3_OUT] = imx_clk_hw_gate("sys_pll3_out", "sys_pll3_bypass", base + 0x114, 11); /* SYS PLL1 fixed output */ - clks[IMX8MN_SYS_PLL1_40M_CG] = imx_clk_gate("sys_pll1_40m_cg", "sys_pll1", base + 0x94, 27); - clks[IMX8MN_SYS_PLL1_80M_CG] = imx_clk_gate("sys_pll1_80m_cg", "sys_pll1", base + 0x94, 25); - clks[IMX8MN_SYS_PLL1_100M_CG] = imx_clk_gate("sys_pll1_100m_cg", "sys_pll1", base + 0x94, 23); - clks[IMX8MN_SYS_PLL1_133M_CG] = imx_clk_gate("sys_pll1_133m_cg", "sys_pll1", base + 0x94, 21); - clks[IMX8MN_SYS_PLL1_160M_CG] = imx_clk_gate("sys_pll1_160m_cg", "sys_pll1", base + 0x94, 19); - clks[IMX8MN_SYS_PLL1_200M_CG] = imx_clk_gate("sys_pll1_200m_cg", "sys_pll1", base + 0x94, 17); - clks[IMX8MN_SYS_PLL1_266M_CG] = imx_clk_gate("sys_pll1_266m_cg", "sys_pll1", base + 0x94, 15); - clks[IMX8MN_SYS_PLL1_400M_CG] = imx_clk_gate("sys_pll1_400m_cg", "sys_pll1", base + 0x94, 13); - clks[IMX8MN_SYS_PLL1_OUT] = imx_clk_gate("sys_pll1_out", "sys_pll1", base + 0x94, 11); - - clks[IMX8MN_SYS_PLL1_40M] = imx_clk_fixed_factor("sys_pll1_40m", "sys_pll1_40m_cg", 1, 20); - clks[IMX8MN_SYS_PLL1_80M] = imx_clk_fixed_factor("sys_pll1_80m", "sys_pll1_80m_cg", 1, 10); - clks[IMX8MN_SYS_PLL1_100M] = imx_clk_fixed_factor("sys_pll1_100m", "sys_pll1_100m_cg", 1, 8); - clks[IMX8MN_SYS_PLL1_133M] = imx_clk_fixed_factor("sys_pll1_133m", "sys_pll1_133m_cg", 1, 6); - clks[IMX8MN_SYS_PLL1_160M] = imx_clk_fixed_factor("sys_pll1_160m", "sys_pll1_160m_cg", 1, 5); - clks[IMX8MN_SYS_PLL1_200M] = imx_clk_fixed_factor("sys_pll1_200m", "sys_pll1_200m_cg", 1, 4); - clks[IMX8MN_SYS_PLL1_266M] = imx_clk_fixed_factor("sys_pll1_266m", "sys_pll1_266m_cg", 1, 3); - clks[IMX8MN_SYS_PLL1_400M] = imx_clk_fixed_factor("sys_pll1_400m", "sys_pll1_400m_cg", 1, 2); - clks[IMX8MN_SYS_PLL1_800M] = imx_clk_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1); + hws[IMX8MN_SYS_PLL1_40M_CG] = imx_clk_hw_gate("sys_pll1_40m_cg", "sys_pll1", base + 0x94, 27); + hws[IMX8MN_SYS_PLL1_80M_CG] = imx_clk_hw_gate("sys_pll1_80m_cg", "sys_pll1", base + 0x94, 25); + hws[IMX8MN_SYS_PLL1_100M_CG] = imx_clk_hw_gate("sys_pll1_100m_cg", "sys_pll1", base + 0x94, 23); + hws[IMX8MN_SYS_PLL1_133M_CG] = imx_clk_hw_gate("sys_pll1_133m_cg", "sys_pll1", base + 0x94, 21); + hws[IMX8MN_SYS_PLL1_160M_CG] = imx_clk_hw_gate("sys_pll1_160m_cg", "sys_pll1", base + 0x94, 19); + hws[IMX8MN_SYS_PLL1_200M_CG] = imx_clk_hw_gate("sys_pll1_200m_cg", "sys_pll1", base + 0x94, 17); + hws[IMX8MN_SYS_PLL1_266M_CG] = imx_clk_hw_gate("sys_pll1_266m_cg", "sys_pll1", base + 0x94, 15); + hws[IMX8MN_SYS_PLL1_400M_CG] = imx_clk_hw_gate("sys_pll1_400m_cg", "sys_pll1", base + 0x94, 13); + hws[IMX8MN_SYS_PLL1_OUT] = imx_clk_hw_gate("sys_pll1_out", "sys_pll1", base + 0x94, 11); + + hws[IMX8MN_SYS_PLL1_40M] = imx_clk_hw_fixed_factor("sys_pll1_40m", "sys_pll1_40m_cg", 1, 20); + hws[IMX8MN_SYS_PLL1_80M] = imx_clk_hw_fixed_factor("sys_pll1_80m", "sys_pll1_80m_cg", 1, 10); + hws[IMX8MN_SYS_PLL1_100M] = imx_clk_hw_fixed_factor("sys_pll1_100m", "sys_pll1_100m_cg", 1, 8); + hws[IMX8MN_SYS_PLL1_133M] = imx_clk_hw_fixed_factor("sys_pll1_133m", "sys_pll1_133m_cg", 1, 6); + hws[IMX8MN_SYS_PLL1_160M] = imx_clk_hw_fixed_factor("sys_pll1_160m", "sys_pll1_160m_cg", 1, 5); + hws[IMX8MN_SYS_PLL1_200M] = imx_clk_hw_fixed_factor("sys_pll1_200m", "sys_pll1_200m_cg", 1, 4); + hws[IMX8MN_SYS_PLL1_266M] = imx_clk_hw_fixed_factor("sys_pll1_266m", "sys_pll1_266m_cg", 1, 3); + hws[IMX8MN_SYS_PLL1_400M] = imx_clk_hw_fixed_factor("sys_pll1_400m", "sys_pll1_400m_cg", 1, 2); + hws[IMX8MN_SYS_PLL1_800M] = imx_clk_hw_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1); /* SYS PLL2 fixed output */ - clks[IMX8MN_SYS_PLL2_50M_CG] = imx_clk_gate("sys_pll2_50m_cg", "sys_pll2", base + 0x104, 27); - clks[IMX8MN_SYS_PLL2_100M_CG] = imx_clk_gate("sys_pll2_100m_cg", "sys_pll2", base + 0x104, 25); - clks[IMX8MN_SYS_PLL2_125M_CG] = imx_clk_gate("sys_pll2_125m_cg", "sys_pll2", base + 0x104, 23); - clks[IMX8MN_SYS_PLL2_166M_CG] = imx_clk_gate("sys_pll2_166m_cg", "sys_pll2", base + 0x104, 21); - clks[IMX8MN_SYS_PLL2_200M_CG] = imx_clk_gate("sys_pll2_200m_cg", "sys_pll2", base + 0x104, 19); - clks[IMX8MN_SYS_PLL2_250M_CG] = imx_clk_gate("sys_pll2_250m_cg", "sys_pll2", base + 0x104, 17); - clks[IMX8MN_SYS_PLL2_333M_CG] = imx_clk_gate("sys_pll2_333m_cg", "sys_pll2", base + 0x104, 15); - clks[IMX8MN_SYS_PLL2_500M_CG] = imx_clk_gate("sys_pll2_500m_cg", "sys_pll2", base + 0x104, 13); - clks[IMX8MN_SYS_PLL2_OUT] = imx_clk_gate("sys_pll2_out", "sys_pll2", base + 0x104, 11); - - clks[IMX8MN_SYS_PLL2_50M] = imx_clk_fixed_factor("sys_pll2_50m", "sys_pll2_50m_cg", 1, 20); - clks[IMX8MN_SYS_PLL2_100M] = imx_clk_fixed_factor("sys_pll2_100m", "sys_pll2_100m_cg", 1, 10); - clks[IMX8MN_SYS_PLL2_125M] = imx_clk_fixed_factor("sys_pll2_125m", "sys_pll2_125m_cg", 1, 8); - clks[IMX8MN_SYS_PLL2_166M] = imx_clk_fixed_factor("sys_pll2_166m", "sys_pll2_166m_cg", 1, 6); - clks[IMX8MN_SYS_PLL2_200M] = imx_clk_fixed_factor("sys_pll2_200m", "sys_pll2_200m_cg", 1, 5); - clks[IMX8MN_SYS_PLL2_250M] = imx_clk_fixed_factor("sys_pll2_250m", "sys_pll2_250m_cg", 1, 4); - clks[IMX8MN_SYS_PLL2_333M] = imx_clk_fixed_factor("sys_pll2_333m", "sys_pll2_333m_cg", 1, 3); - clks[IMX8MN_SYS_PLL2_500M] = imx_clk_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); - clks[IMX8MN_SYS_PLL2_1000M] = imx_clk_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + hws[IMX8MN_SYS_PLL2_50M_CG] = imx_clk_hw_gate("sys_pll2_50m_cg", "sys_pll2", base + 0x104, 27); + hws[IMX8MN_SYS_PLL2_100M_CG] = imx_clk_hw_gate("sys_pll2_100m_cg", "sys_pll2", base + 0x104, 25); + hws[IMX8MN_SYS_PLL2_125M_CG] = imx_clk_hw_gate("sys_pll2_125m_cg", "sys_pll2", base + 0x104, 23); + hws[IMX8MN_SYS_PLL2_166M_CG] = imx_clk_hw_gate("sys_pll2_166m_cg", "sys_pll2", base + 0x104, 21); + hws[IMX8MN_SYS_PLL2_200M_CG] = imx_clk_hw_gate("sys_pll2_200m_cg", "sys_pll2", base + 0x104, 19); + hws[IMX8MN_SYS_PLL2_250M_CG] = imx_clk_hw_gate("sys_pll2_250m_cg", "sys_pll2", base + 0x104, 17); + hws[IMX8MN_SYS_PLL2_333M_CG] = imx_clk_hw_gate("sys_pll2_333m_cg", "sys_pll2", base + 0x104, 15); + hws[IMX8MN_SYS_PLL2_500M_CG] = imx_clk_hw_gate("sys_pll2_500m_cg", "sys_pll2", base + 0x104, 13); + hws[IMX8MN_SYS_PLL2_OUT] = imx_clk_hw_gate("sys_pll2_out", "sys_pll2", base + 0x104, 11); + + hws[IMX8MN_SYS_PLL2_50M] = imx_clk_hw_fixed_factor("sys_pll2_50m", "sys_pll2_50m_cg", 1, 20); + hws[IMX8MN_SYS_PLL2_100M] = imx_clk_hw_fixed_factor("sys_pll2_100m", "sys_pll2_100m_cg", 1, 10); + hws[IMX8MN_SYS_PLL2_125M] = imx_clk_hw_fixed_factor("sys_pll2_125m", "sys_pll2_125m_cg", 1, 8); + hws[IMX8MN_SYS_PLL2_166M] = imx_clk_hw_fixed_factor("sys_pll2_166m", "sys_pll2_166m_cg", 1, 6); + hws[IMX8MN_SYS_PLL2_200M] = imx_clk_hw_fixed_factor("sys_pll2_200m", "sys_pll2_200m_cg", 1, 5); + hws[IMX8MN_SYS_PLL2_250M] = imx_clk_hw_fixed_factor("sys_pll2_250m", "sys_pll2_250m_cg", 1, 4); + hws[IMX8MN_SYS_PLL2_333M] = imx_clk_hw_fixed_factor("sys_pll2_333m", "sys_pll2_333m_cg", 1, 3); + hws[IMX8MN_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); + hws[IMX8MN_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); if (WARN_ON(IS_ERR(base))) { ret = PTR_ERR(base); - goto unregister_clks; + goto unregister_hws; } /* CORE */ - clks[IMX8MN_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mn_a53_sels, ARRAY_SIZE(imx8mn_a53_sels)); - clks[IMX8MN_CLK_GPU_CORE_SRC] = imx_clk_mux2("gpu_core_src", base + 0x8180, 24, 3, imx8mn_gpu_core_sels, ARRAY_SIZE(imx8mn_gpu_core_sels)); - clks[IMX8MN_CLK_GPU_SHADER_SRC] = imx_clk_mux2("gpu_shader_src", base + 0x8200, 24, 3, imx8mn_gpu_shader_sels, ARRAY_SIZE(imx8mn_gpu_shader_sels)); - clks[IMX8MN_CLK_A53_CG] = imx_clk_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28); - clks[IMX8MN_CLK_GPU_CORE_CG] = imx_clk_gate3("gpu_core_cg", "gpu_core_src", base + 0x8180, 28); - clks[IMX8MN_CLK_GPU_SHADER_CG] = imx_clk_gate3("gpu_shader_cg", "gpu_shader_src", base + 0x8200, 28); + hws[IMX8MN_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mn_a53_sels, ARRAY_SIZE(imx8mn_a53_sels)); + hws[IMX8MN_CLK_GPU_CORE_SRC] = imx_clk_hw_mux2("gpu_core_src", base + 0x8180, 24, 3, imx8mn_gpu_core_sels, ARRAY_SIZE(imx8mn_gpu_core_sels)); + hws[IMX8MN_CLK_GPU_SHADER_SRC] = imx_clk_hw_mux2("gpu_shader_src", base + 0x8200, 24, 3, imx8mn_gpu_shader_sels, ARRAY_SIZE(imx8mn_gpu_shader_sels)); + hws[IMX8MN_CLK_A53_CG] = imx_clk_hw_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28); + hws[IMX8MN_CLK_GPU_CORE_CG] = imx_clk_hw_gate3("gpu_core_cg", "gpu_core_src", base + 0x8180, 28); + hws[IMX8MN_CLK_GPU_SHADER_CG] = imx_clk_hw_gate3("gpu_shader_cg", "gpu_shader_src", base + 0x8200, 28); - clks[IMX8MN_CLK_A53_DIV] = imx_clk_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); - clks[IMX8MN_CLK_GPU_CORE_DIV] = imx_clk_divider2("gpu_core_div", "gpu_core_cg", base + 0x8180, 0, 3); - clks[IMX8MN_CLK_GPU_SHADER_DIV] = imx_clk_divider2("gpu_shader_div", "gpu_shader_cg", base + 0x8200, 0, 3); + hws[IMX8MN_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); + hws[IMX8MN_CLK_GPU_CORE_DIV] = imx_clk_hw_divider2("gpu_core_div", "gpu_core_cg", base + 0x8180, 0, 3); + hws[IMX8MN_CLK_GPU_SHADER_DIV] = imx_clk_hw_divider2("gpu_shader_div", "gpu_shader_cg", base + 0x8200, 0, 3); /* BUS */ - clks[IMX8MN_CLK_MAIN_AXI] = imx8m_clk_composite_critical("main_axi", imx8mn_main_axi_sels, base + 0x8800); - clks[IMX8MN_CLK_ENET_AXI] = imx8m_clk_composite("enet_axi", imx8mn_enet_axi_sels, base + 0x8880); - clks[IMX8MN_CLK_NAND_USDHC_BUS] = imx8m_clk_composite("nand_usdhc_bus", imx8mn_nand_usdhc_sels, base + 0x8900); - clks[IMX8MN_CLK_DISP_AXI] = imx8m_clk_composite("disp_axi", imx8mn_disp_axi_sels, base + 0x8a00); - clks[IMX8MN_CLK_DISP_APB] = imx8m_clk_composite("disp_apb", imx8mn_disp_apb_sels, base + 0x8a80); - clks[IMX8MN_CLK_USB_BUS] = imx8m_clk_composite("usb_bus", imx8mn_usb_bus_sels, base + 0x8b80); - clks[IMX8MN_CLK_GPU_AXI] = imx8m_clk_composite("gpu_axi", imx8mn_gpu_axi_sels, base + 0x8c00); - clks[IMX8MN_CLK_GPU_AHB] = imx8m_clk_composite("gpu_ahb", imx8mn_gpu_ahb_sels, base + 0x8c80); - clks[IMX8MN_CLK_NOC] = imx8m_clk_composite_critical("noc", imx8mn_noc_sels, base + 0x8d00); - - clks[IMX8MN_CLK_AHB] = imx8m_clk_composite_critical("ahb", imx8mn_ahb_sels, base + 0x9000); - clks[IMX8MN_CLK_AUDIO_AHB] = imx8m_clk_composite("audio_ahb", imx8mn_audio_ahb_sels, base + 0x9100); - clks[IMX8MN_CLK_IPG_ROOT] = imx_clk_divider2("ipg_root", "ahb", base + 0x9080, 0, 1); - clks[IMX8MN_CLK_IPG_AUDIO_ROOT] = imx_clk_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1); - clks[IMX8MN_CLK_DRAM_CORE] = imx_clk_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mn_dram_core_sels, ARRAY_SIZE(imx8mn_dram_core_sels), CLK_IS_CRITICAL); - clks[IMX8MN_CLK_DRAM_ALT] = imx8m_clk_composite("dram_alt", imx8mn_dram_alt_sels, base + 0xa000); - clks[IMX8MN_CLK_DRAM_APB] = imx8m_clk_composite_critical("dram_apb", imx8mn_dram_apb_sels, base + 0xa080); - clks[IMX8MN_CLK_DISP_PIXEL] = imx8m_clk_composite("disp_pixel", imx8mn_disp_pixel_sels, base + 0xa500); - clks[IMX8MN_CLK_SAI2] = imx8m_clk_composite("sai2", imx8mn_sai2_sels, base + 0xa600); - clks[IMX8MN_CLK_SAI3] = imx8m_clk_composite("sai3", imx8mn_sai3_sels, base + 0xa680); - clks[IMX8MN_CLK_SAI5] = imx8m_clk_composite("sai5", imx8mn_sai5_sels, base + 0xa780); - clks[IMX8MN_CLK_SAI6] = imx8m_clk_composite("sai6", imx8mn_sai6_sels, base + 0xa800); - clks[IMX8MN_CLK_SPDIF1] = imx8m_clk_composite("spdif1", imx8mn_spdif1_sels, base + 0xa880); - clks[IMX8MN_CLK_ENET_REF] = imx8m_clk_composite("enet_ref", imx8mn_enet_ref_sels, base + 0xa980); - clks[IMX8MN_CLK_ENET_TIMER] = imx8m_clk_composite("enet_timer", imx8mn_enet_timer_sels, base + 0xaa00); - clks[IMX8MN_CLK_ENET_PHY_REF] = imx8m_clk_composite("enet_phy", imx8mn_enet_phy_sels, base + 0xaa80); - clks[IMX8MN_CLK_NAND] = imx8m_clk_composite("nand", imx8mn_nand_sels, base + 0xab00); - clks[IMX8MN_CLK_QSPI] = imx8m_clk_composite("qspi", imx8mn_qspi_sels, base + 0xab80); - clks[IMX8MN_CLK_USDHC1] = imx8m_clk_composite("usdhc1", imx8mn_usdhc1_sels, base + 0xac00); - clks[IMX8MN_CLK_USDHC2] = imx8m_clk_composite("usdhc2", imx8mn_usdhc2_sels, base + 0xac80); - clks[IMX8MN_CLK_I2C1] = imx8m_clk_composite("i2c1", imx8mn_i2c1_sels, base + 0xad00); - clks[IMX8MN_CLK_I2C2] = imx8m_clk_composite("i2c2", imx8mn_i2c2_sels, base + 0xad80); - clks[IMX8MN_CLK_I2C3] = imx8m_clk_composite("i2c3", imx8mn_i2c3_sels, base + 0xae00); - clks[IMX8MN_CLK_I2C4] = imx8m_clk_composite("i2c4", imx8mn_i2c4_sels, base + 0xae80); - clks[IMX8MN_CLK_UART1] = imx8m_clk_composite("uart1", imx8mn_uart1_sels, base + 0xaf00); - clks[IMX8MN_CLK_UART2] = imx8m_clk_composite("uart2", imx8mn_uart2_sels, base + 0xaf80); - clks[IMX8MN_CLK_UART3] = imx8m_clk_composite("uart3", imx8mn_uart3_sels, base + 0xb000); - clks[IMX8MN_CLK_UART4] = imx8m_clk_composite("uart4", imx8mn_uart4_sels, base + 0xb080); - clks[IMX8MN_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mn_usb_core_sels, base + 0xb100); - clks[IMX8MN_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mn_usb_phy_sels, base + 0xb180); - clks[IMX8MN_CLK_GIC] = imx8m_clk_composite_critical("gic", imx8mn_gic_sels, base + 0xb200); - clks[IMX8MN_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mn_ecspi1_sels, base + 0xb280); - clks[IMX8MN_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mn_ecspi2_sels, base + 0xb300); - clks[IMX8MN_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mn_pwm1_sels, base + 0xb380); - clks[IMX8MN_CLK_PWM2] = imx8m_clk_composite("pwm2", imx8mn_pwm2_sels, base + 0xb400); - clks[IMX8MN_CLK_PWM3] = imx8m_clk_composite("pwm3", imx8mn_pwm3_sels, base + 0xb480); - clks[IMX8MN_CLK_PWM4] = imx8m_clk_composite("pwm4", imx8mn_pwm4_sels, base + 0xb500); - clks[IMX8MN_CLK_WDOG] = imx8m_clk_composite("wdog", imx8mn_wdog_sels, base + 0xb900); - clks[IMX8MN_CLK_WRCLK] = imx8m_clk_composite("wrclk", imx8mn_wrclk_sels, base + 0xb980); - clks[IMX8MN_CLK_CLKO1] = imx8m_clk_composite("clko1", imx8mn_clko1_sels, base + 0xba00); - clks[IMX8MN_CLK_CLKO2] = imx8m_clk_composite("clko2", imx8mn_clko2_sels, base + 0xba80); - clks[IMX8MN_CLK_DSI_CORE] = imx8m_clk_composite("dsi_core", imx8mn_dsi_core_sels, base + 0xbb00); - clks[IMX8MN_CLK_DSI_PHY_REF] = imx8m_clk_composite("dsi_phy_ref", imx8mn_dsi_phy_sels, base + 0xbb80); - clks[IMX8MN_CLK_DSI_DBI] = imx8m_clk_composite("dsi_dbi", imx8mn_dsi_dbi_sels, base + 0xbc00); - clks[IMX8MN_CLK_USDHC3] = imx8m_clk_composite("usdhc3", imx8mn_usdhc3_sels, base + 0xbc80); - clks[IMX8MN_CLK_CAMERA_PIXEL] = imx8m_clk_composite("camera_pixel", imx8mn_camera_pixel_sels, base + 0xbd00); - clks[IMX8MN_CLK_CSI1_PHY_REF] = imx8m_clk_composite("csi1_phy_ref", imx8mn_csi1_phy_sels, base + 0xbd80); - clks[IMX8MN_CLK_CSI2_PHY_REF] = imx8m_clk_composite("csi2_phy_ref", imx8mn_csi2_phy_sels, base + 0xbf00); - clks[IMX8MN_CLK_CSI2_ESC] = imx8m_clk_composite("csi2_esc", imx8mn_csi2_esc_sels, base + 0xbf80); - clks[IMX8MN_CLK_ECSPI3] = imx8m_clk_composite("ecspi3", imx8mn_ecspi3_sels, base + 0xc180); - clks[IMX8MN_CLK_PDM] = imx8m_clk_composite("pdm", imx8mn_pdm_sels, base + 0xc200); - clks[IMX8MN_CLK_SAI7] = imx8m_clk_composite("sai7", imx8mn_sai7_sels, base + 0xc300); - - clks[IMX8MN_CLK_ECSPI1_ROOT] = imx_clk_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0); - clks[IMX8MN_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); - clks[IMX8MN_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); - clks[IMX8MN_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); - clks[IMX8MN_CLK_GPIO1_ROOT] = imx_clk_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); - clks[IMX8MN_CLK_GPIO2_ROOT] = imx_clk_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); - clks[IMX8MN_CLK_GPIO3_ROOT] = imx_clk_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); - clks[IMX8MN_CLK_GPIO4_ROOT] = imx_clk_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); - clks[IMX8MN_CLK_GPIO5_ROOT] = imx_clk_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); - clks[IMX8MN_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); - clks[IMX8MN_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); - clks[IMX8MN_CLK_I2C3_ROOT] = imx_clk_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0); - clks[IMX8MN_CLK_I2C4_ROOT] = imx_clk_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0); - clks[IMX8MN_CLK_MU_ROOT] = imx_clk_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0); - clks[IMX8MN_CLK_OCOTP_ROOT] = imx_clk_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0); - clks[IMX8MN_CLK_PWM1_ROOT] = imx_clk_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0); - clks[IMX8MN_CLK_PWM2_ROOT] = imx_clk_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0); - clks[IMX8MN_CLK_PWM3_ROOT] = imx_clk_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0); - clks[IMX8MN_CLK_PWM4_ROOT] = imx_clk_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0); - clks[IMX8MN_CLK_QSPI_ROOT] = imx_clk_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0); - clks[IMX8MN_CLK_NAND_ROOT] = imx_clk_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand); - clks[IMX8MN_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand); - clks[IMX8MN_CLK_SAI2_ROOT] = imx_clk_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2); - clks[IMX8MN_CLK_SAI2_IPG] = imx_clk_gate2_shared2("sai2_ipg_clk", "ipg_audio_root", base + 0x4340, 0, &share_count_sai2); - clks[IMX8MN_CLK_SAI3_ROOT] = imx_clk_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3); - clks[IMX8MN_CLK_SAI3_IPG] = imx_clk_gate2_shared2("sai3_ipg_clk", "ipg_audio_root", base + 0x4350, 0, &share_count_sai3); - clks[IMX8MN_CLK_SAI5_ROOT] = imx_clk_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5); - clks[IMX8MN_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); - clks[IMX8MN_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); - clks[IMX8MN_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); - clks[IMX8MN_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); - clks[IMX8MN_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); - clks[IMX8MN_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); - clks[IMX8MN_CLK_UART4_ROOT] = imx_clk_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0); - clks[IMX8MN_CLK_USB1_CTRL_ROOT] = imx_clk_gate4("usb1_ctrl_root_clk", "usb_core_ref", base + 0x44d0, 0); - clks[IMX8MN_CLK_GPU_CORE_ROOT] = imx_clk_gate4("gpu_core_root_clk", "gpu_core_div", base + 0x44f0, 0); - clks[IMX8MN_CLK_USDHC1_ROOT] = imx_clk_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0); - clks[IMX8MN_CLK_USDHC2_ROOT] = imx_clk_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0); - clks[IMX8MN_CLK_WDOG1_ROOT] = imx_clk_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0); - clks[IMX8MN_CLK_WDOG2_ROOT] = imx_clk_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0); - clks[IMX8MN_CLK_WDOG3_ROOT] = imx_clk_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0); - clks[IMX8MN_CLK_GPU_BUS_ROOT] = imx_clk_gate4("gpu_root_clk", "gpu_axi", base + 0x4570, 0); - clks[IMX8MN_CLK_ASRC_ROOT] = imx_clk_gate4("asrc_root_clk", "audio_ahb", base + 0x4580, 0); - clks[IMX8MN_CLK_PDM_ROOT] = imx_clk_gate2_shared2("pdm_root_clk", "pdm", base + 0x45b0, 0, &share_count_pdm); - clks[IMX8MN_CLK_PDM_IPG] = imx_clk_gate2_shared2("pdm_ipg_clk", "ipg_audio_root", base + 0x45b0, 0, &share_count_pdm); - clks[IMX8MN_CLK_DISP_AXI_ROOT] = imx_clk_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MN_CLK_DISP_APB_ROOT] = imx_clk_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MN_CLK_CAMERA_PIXEL_ROOT] = imx_clk_gate2_shared2("camera_pixel_clk", "camera_pixel", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MN_CLK_DISP_PIXEL_ROOT] = imx_clk_gate2_shared2("disp_pixel_clk", "disp_pixel", base + 0x45d0, 0, &share_count_disp); - clks[IMX8MN_CLK_USDHC3_ROOT] = imx_clk_gate4("usdhc3_root_clk", "usdhc3", base + 0x45e0, 0); - clks[IMX8MN_CLK_TMU_ROOT] = imx_clk_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0); - clks[IMX8MN_CLK_SDMA1_ROOT] = imx_clk_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0); - clks[IMX8MN_CLK_SDMA2_ROOT] = imx_clk_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0); - clks[IMX8MN_CLK_SDMA3_ROOT] = imx_clk_gate4("sdma3_clk", "ipg_audio_root", base + 0x45f0, 0); - clks[IMX8MN_CLK_SAI7_ROOT] = imx_clk_gate2_shared2("sai7_root_clk", "sai7", base + 0x4650, 0, &share_count_sai7); - - clks[IMX8MN_CLK_DRAM_ALT_ROOT] = imx_clk_fixed_factor("dram_alt_root", "dram_alt", 1, 4); - - clks[IMX8MN_CLK_ARM] = imx_clk_cpu("arm", "arm_a53_div", - clks[IMX8MN_CLK_A53_DIV], - clks[IMX8MN_CLK_A53_SRC], - clks[IMX8MN_ARM_PLL_OUT], - clks[IMX8MN_SYS_PLL1_800M]); - - imx_check_clocks(clks, ARRAY_SIZE(clks)); - - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + hws[IMX8MN_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi", imx8mn_main_axi_sels, base + 0x8800); + hws[IMX8MN_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mn_enet_axi_sels, base + 0x8880); + hws[IMX8MN_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite("nand_usdhc_bus", imx8mn_nand_usdhc_sels, base + 0x8900); + hws[IMX8MN_CLK_DISP_AXI] = imx8m_clk_hw_composite("disp_axi", imx8mn_disp_axi_sels, base + 0x8a00); + hws[IMX8MN_CLK_DISP_APB] = imx8m_clk_hw_composite("disp_apb", imx8mn_disp_apb_sels, base + 0x8a80); + hws[IMX8MN_CLK_USB_BUS] = imx8m_clk_hw_composite("usb_bus", imx8mn_usb_bus_sels, base + 0x8b80); + hws[IMX8MN_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mn_gpu_axi_sels, base + 0x8c00); + hws[IMX8MN_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mn_gpu_ahb_sels, base + 0x8c80); + hws[IMX8MN_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mn_noc_sels, base + 0x8d00); + + hws[IMX8MN_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb", imx8mn_ahb_sels, base + 0x9000); + hws[IMX8MN_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mn_audio_ahb_sels, base + 0x9100); + hws[IMX8MN_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb", base + 0x9080, 0, 1); + hws[IMX8MN_CLK_IPG_AUDIO_ROOT] = imx_clk_hw_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1); + hws[IMX8MN_CLK_DRAM_CORE] = imx_clk_hw_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mn_dram_core_sels, ARRAY_SIZE(imx8mn_dram_core_sels), CLK_IS_CRITICAL); + + /* + * DRAM clocks are manipulated from TF-A outside clock framework. + * Mark with GET_RATE_NOCACHE to always read div value from hardware + */ + hws[IMX8MN_CLK_DRAM_ALT] = __imx8m_clk_hw_composite("dram_alt", imx8mn_dram_alt_sels, base + 0xa000, CLK_GET_RATE_NOCACHE); + hws[IMX8MN_CLK_DRAM_APB] = __imx8m_clk_hw_composite("dram_apb", imx8mn_dram_apb_sels, base + 0xa080, CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); + + hws[IMX8MN_CLK_DISP_PIXEL] = imx8m_clk_hw_composite("disp_pixel", imx8mn_disp_pixel_sels, base + 0xa500); + hws[IMX8MN_CLK_SAI2] = imx8m_clk_hw_composite("sai2", imx8mn_sai2_sels, base + 0xa600); + hws[IMX8MN_CLK_SAI3] = imx8m_clk_hw_composite("sai3", imx8mn_sai3_sels, base + 0xa680); + hws[IMX8MN_CLK_SAI5] = imx8m_clk_hw_composite("sai5", imx8mn_sai5_sels, base + 0xa780); + hws[IMX8MN_CLK_SAI6] = imx8m_clk_hw_composite("sai6", imx8mn_sai6_sels, base + 0xa800); + hws[IMX8MN_CLK_SPDIF1] = imx8m_clk_hw_composite("spdif1", imx8mn_spdif1_sels, base + 0xa880); + hws[IMX8MN_CLK_ENET_REF] = imx8m_clk_hw_composite("enet_ref", imx8mn_enet_ref_sels, base + 0xa980); + hws[IMX8MN_CLK_ENET_TIMER] = imx8m_clk_hw_composite("enet_timer", imx8mn_enet_timer_sels, base + 0xaa00); + hws[IMX8MN_CLK_ENET_PHY_REF] = imx8m_clk_hw_composite("enet_phy", imx8mn_enet_phy_sels, base + 0xaa80); + hws[IMX8MN_CLK_NAND] = imx8m_clk_hw_composite("nand", imx8mn_nand_sels, base + 0xab00); + hws[IMX8MN_CLK_QSPI] = imx8m_clk_hw_composite("qspi", imx8mn_qspi_sels, base + 0xab80); + hws[IMX8MN_CLK_USDHC1] = imx8m_clk_hw_composite("usdhc1", imx8mn_usdhc1_sels, base + 0xac00); + hws[IMX8MN_CLK_USDHC2] = imx8m_clk_hw_composite("usdhc2", imx8mn_usdhc2_sels, base + 0xac80); + hws[IMX8MN_CLK_I2C1] = imx8m_clk_hw_composite("i2c1", imx8mn_i2c1_sels, base + 0xad00); + hws[IMX8MN_CLK_I2C2] = imx8m_clk_hw_composite("i2c2", imx8mn_i2c2_sels, base + 0xad80); + hws[IMX8MN_CLK_I2C3] = imx8m_clk_hw_composite("i2c3", imx8mn_i2c3_sels, base + 0xae00); + hws[IMX8MN_CLK_I2C4] = imx8m_clk_hw_composite("i2c4", imx8mn_i2c4_sels, base + 0xae80); + hws[IMX8MN_CLK_UART1] = imx8m_clk_hw_composite("uart1", imx8mn_uart1_sels, base + 0xaf00); + hws[IMX8MN_CLK_UART2] = imx8m_clk_hw_composite("uart2", imx8mn_uart2_sels, base + 0xaf80); + hws[IMX8MN_CLK_UART3] = imx8m_clk_hw_composite("uart3", imx8mn_uart3_sels, base + 0xb000); + hws[IMX8MN_CLK_UART4] = imx8m_clk_hw_composite("uart4", imx8mn_uart4_sels, base + 0xb080); + hws[IMX8MN_CLK_USB_CORE_REF] = imx8m_clk_hw_composite("usb_core_ref", imx8mn_usb_core_sels, base + 0xb100); + hws[IMX8MN_CLK_USB_PHY_REF] = imx8m_clk_hw_composite("usb_phy_ref", imx8mn_usb_phy_sels, base + 0xb180); + hws[IMX8MN_CLK_GIC] = imx8m_clk_hw_composite_critical("gic", imx8mn_gic_sels, base + 0xb200); + hws[IMX8MN_CLK_ECSPI1] = imx8m_clk_hw_composite("ecspi1", imx8mn_ecspi1_sels, base + 0xb280); + hws[IMX8MN_CLK_ECSPI2] = imx8m_clk_hw_composite("ecspi2", imx8mn_ecspi2_sels, base + 0xb300); + hws[IMX8MN_CLK_PWM1] = imx8m_clk_hw_composite("pwm1", imx8mn_pwm1_sels, base + 0xb380); + hws[IMX8MN_CLK_PWM2] = imx8m_clk_hw_composite("pwm2", imx8mn_pwm2_sels, base + 0xb400); + hws[IMX8MN_CLK_PWM3] = imx8m_clk_hw_composite("pwm3", imx8mn_pwm3_sels, base + 0xb480); + hws[IMX8MN_CLK_PWM4] = imx8m_clk_hw_composite("pwm4", imx8mn_pwm4_sels, base + 0xb500); + hws[IMX8MN_CLK_WDOG] = imx8m_clk_hw_composite("wdog", imx8mn_wdog_sels, base + 0xb900); + hws[IMX8MN_CLK_WRCLK] = imx8m_clk_hw_composite("wrclk", imx8mn_wrclk_sels, base + 0xb980); + hws[IMX8MN_CLK_CLKO1] = imx8m_clk_hw_composite("clko1", imx8mn_clko1_sels, base + 0xba00); + hws[IMX8MN_CLK_CLKO2] = imx8m_clk_hw_composite("clko2", imx8mn_clko2_sels, base + 0xba80); + hws[IMX8MN_CLK_DSI_CORE] = imx8m_clk_hw_composite("dsi_core", imx8mn_dsi_core_sels, base + 0xbb00); + hws[IMX8MN_CLK_DSI_PHY_REF] = imx8m_clk_hw_composite("dsi_phy_ref", imx8mn_dsi_phy_sels, base + 0xbb80); + hws[IMX8MN_CLK_DSI_DBI] = imx8m_clk_hw_composite("dsi_dbi", imx8mn_dsi_dbi_sels, base + 0xbc00); + hws[IMX8MN_CLK_USDHC3] = imx8m_clk_hw_composite("usdhc3", imx8mn_usdhc3_sels, base + 0xbc80); + hws[IMX8MN_CLK_CAMERA_PIXEL] = imx8m_clk_hw_composite("camera_pixel", imx8mn_camera_pixel_sels, base + 0xbd00); + hws[IMX8MN_CLK_CSI1_PHY_REF] = imx8m_clk_hw_composite("csi1_phy_ref", imx8mn_csi1_phy_sels, base + 0xbd80); + hws[IMX8MN_CLK_CSI2_PHY_REF] = imx8m_clk_hw_composite("csi2_phy_ref", imx8mn_csi2_phy_sels, base + 0xbf00); + hws[IMX8MN_CLK_CSI2_ESC] = imx8m_clk_hw_composite("csi2_esc", imx8mn_csi2_esc_sels, base + 0xbf80); + hws[IMX8MN_CLK_ECSPI3] = imx8m_clk_hw_composite("ecspi3", imx8mn_ecspi3_sels, base + 0xc180); + hws[IMX8MN_CLK_PDM] = imx8m_clk_hw_composite("pdm", imx8mn_pdm_sels, base + 0xc200); + hws[IMX8MN_CLK_SAI7] = imx8m_clk_hw_composite("sai7", imx8mn_sai7_sels, base + 0xc300); + + hws[IMX8MN_CLK_ECSPI1_ROOT] = imx_clk_hw_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0); + hws[IMX8MN_CLK_ECSPI2_ROOT] = imx_clk_hw_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); + hws[IMX8MN_CLK_ECSPI3_ROOT] = imx_clk_hw_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); + hws[IMX8MN_CLK_ENET1_ROOT] = imx_clk_hw_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); + hws[IMX8MN_CLK_GPIO1_ROOT] = imx_clk_hw_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); + hws[IMX8MN_CLK_GPIO2_ROOT] = imx_clk_hw_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); + hws[IMX8MN_CLK_GPIO3_ROOT] = imx_clk_hw_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); + hws[IMX8MN_CLK_GPIO4_ROOT] = imx_clk_hw_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); + hws[IMX8MN_CLK_GPIO5_ROOT] = imx_clk_hw_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); + hws[IMX8MN_CLK_I2C1_ROOT] = imx_clk_hw_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); + hws[IMX8MN_CLK_I2C2_ROOT] = imx_clk_hw_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); + hws[IMX8MN_CLK_I2C3_ROOT] = imx_clk_hw_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0); + hws[IMX8MN_CLK_I2C4_ROOT] = imx_clk_hw_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0); + hws[IMX8MN_CLK_MU_ROOT] = imx_clk_hw_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0); + hws[IMX8MN_CLK_OCOTP_ROOT] = imx_clk_hw_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0); + hws[IMX8MN_CLK_PWM1_ROOT] = imx_clk_hw_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0); + hws[IMX8MN_CLK_PWM2_ROOT] = imx_clk_hw_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0); + hws[IMX8MN_CLK_PWM3_ROOT] = imx_clk_hw_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0); + hws[IMX8MN_CLK_PWM4_ROOT] = imx_clk_hw_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0); + hws[IMX8MN_CLK_QSPI_ROOT] = imx_clk_hw_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0); + hws[IMX8MN_CLK_NAND_ROOT] = imx_clk_hw_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand); + hws[IMX8MN_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand); + hws[IMX8MN_CLK_SAI2_ROOT] = imx_clk_hw_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2); + hws[IMX8MN_CLK_SAI2_IPG] = imx_clk_hw_gate2_shared2("sai2_ipg_clk", "ipg_audio_root", base + 0x4340, 0, &share_count_sai2); + hws[IMX8MN_CLK_SAI3_ROOT] = imx_clk_hw_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3); + hws[IMX8MN_CLK_SAI3_IPG] = imx_clk_hw_gate2_shared2("sai3_ipg_clk", "ipg_audio_root", base + 0x4350, 0, &share_count_sai3); + hws[IMX8MN_CLK_SAI5_ROOT] = imx_clk_hw_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5); + hws[IMX8MN_CLK_SAI5_IPG] = imx_clk_hw_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); + hws[IMX8MN_CLK_SAI6_ROOT] = imx_clk_hw_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); + hws[IMX8MN_CLK_SAI6_IPG] = imx_clk_hw_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); + hws[IMX8MN_CLK_UART1_ROOT] = imx_clk_hw_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); + hws[IMX8MN_CLK_UART2_ROOT] = imx_clk_hw_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); + hws[IMX8MN_CLK_UART3_ROOT] = imx_clk_hw_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); + hws[IMX8MN_CLK_UART4_ROOT] = imx_clk_hw_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0); + hws[IMX8MN_CLK_USB1_CTRL_ROOT] = imx_clk_hw_gate4("usb1_ctrl_root_clk", "usb_bus", base + 0x44d0, 0); + hws[IMX8MN_CLK_GPU_CORE_ROOT] = imx_clk_hw_gate4("gpu_core_root_clk", "gpu_core_div", base + 0x44f0, 0); + hws[IMX8MN_CLK_USDHC1_ROOT] = imx_clk_hw_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0); + hws[IMX8MN_CLK_USDHC2_ROOT] = imx_clk_hw_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0); + hws[IMX8MN_CLK_WDOG1_ROOT] = imx_clk_hw_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0); + hws[IMX8MN_CLK_WDOG2_ROOT] = imx_clk_hw_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0); + hws[IMX8MN_CLK_WDOG3_ROOT] = imx_clk_hw_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0); + hws[IMX8MN_CLK_GPU_BUS_ROOT] = imx_clk_hw_gate4("gpu_root_clk", "gpu_axi", base + 0x4570, 0); + hws[IMX8MN_CLK_ASRC_ROOT] = imx_clk_hw_gate4("asrc_root_clk", "audio_ahb", base + 0x4580, 0); + hws[IMX8MN_CLK_PDM_ROOT] = imx_clk_hw_gate2_shared2("pdm_root_clk", "pdm", base + 0x45b0, 0, &share_count_pdm); + hws[IMX8MN_CLK_PDM_IPG] = imx_clk_hw_gate2_shared2("pdm_ipg_clk", "ipg_audio_root", base + 0x45b0, 0, &share_count_pdm); + hws[IMX8MN_CLK_DISP_AXI_ROOT] = imx_clk_hw_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MN_CLK_DISP_APB_ROOT] = imx_clk_hw_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MN_CLK_CAMERA_PIXEL_ROOT] = imx_clk_hw_gate2_shared2("camera_pixel_clk", "camera_pixel", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MN_CLK_DISP_PIXEL_ROOT] = imx_clk_hw_gate2_shared2("disp_pixel_clk", "disp_pixel", base + 0x45d0, 0, &share_count_disp); + hws[IMX8MN_CLK_USDHC3_ROOT] = imx_clk_hw_gate4("usdhc3_root_clk", "usdhc3", base + 0x45e0, 0); + hws[IMX8MN_CLK_TMU_ROOT] = imx_clk_hw_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0); + hws[IMX8MN_CLK_SDMA1_ROOT] = imx_clk_hw_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0); + hws[IMX8MN_CLK_SDMA2_ROOT] = imx_clk_hw_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0); + hws[IMX8MN_CLK_SDMA3_ROOT] = imx_clk_hw_gate4("sdma3_clk", "ipg_audio_root", base + 0x45f0, 0); + hws[IMX8MN_CLK_SAI7_ROOT] = imx_clk_hw_gate2_shared2("sai7_root_clk", "sai7", base + 0x4650, 0, &share_count_sai7); + + hws[IMX8MN_CLK_DRAM_ALT_ROOT] = imx_clk_hw_fixed_factor("dram_alt_root", "dram_alt", 1, 4); + + hws[IMX8MN_CLK_ARM] = imx_clk_hw_cpu("arm", "arm_a53_div", + hws[IMX8MN_CLK_A53_DIV]->clk, + hws[IMX8MN_CLK_A53_SRC]->clk, + hws[IMX8MN_ARM_PLL_OUT]->clk, + hws[IMX8MN_SYS_PLL1_800M]->clk); + + imx_check_clk_hws(hws, IMX8MN_CLK_END); + + ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); if (ret < 0) { - dev_err(dev, "failed to register clks for i.MX8MN\n"); - goto unregister_clks; + dev_err(dev, "failed to register hws for i.MX8MN\n"); + goto unregister_hws; + } + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_hws[i] = &hws[index]->clk; } - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(uart_hws); return 0; -unregister_clks: - imx_unregister_clocks(clks, ARRAY_SIZE(clks)); +unregister_hws: + imx_unregister_hw_clocks(hws, IMX8MN_CLK_END); return ret; } @@ -572,6 +591,11 @@ static struct platform_driver imx8mn_clk_driver = { .probe = imx8mn_clocks_probe, .driver = { .name = "imx8mn-ccm", + /* + * Disable bind attributes: clocks are not removed and + * reloading the driver will crash or break devices. + */ + .suppress_bind_attrs = true, .of_match_table = of_match_ptr(imx8mn_clk_of_match), }, }; diff --git a/drivers/clk/imx/clk-imx8mp.c b/drivers/clk/imx/clk-imx8mp.c new file mode 100644 index 000000000000..f6c120cca0d4 --- /dev/null +++ b/drivers/clk/imx/clk-imx8mp.c @@ -0,0 +1,764 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 NXP. + */ + +#include <dt-bindings/clock/imx8mp-clock.h> +#include <linux/clkdev.h> +#include <linux/clk-provider.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +#include "clk.h" + +static u32 share_count_nand; +static u32 share_count_media; + +static const char * const pll_ref_sels[] = { "osc_24m", "dummy", "dummy", "dummy", }; +static const char * const audio_pll1_bypass_sels[] = {"audio_pll1", "audio_pll1_ref_sel", }; +static const char * const audio_pll2_bypass_sels[] = {"audio_pll2", "audio_pll2_ref_sel", }; +static const char * const video_pll1_bypass_sels[] = {"video_pll1", "video_pll1_ref_sel", }; +static const char * const dram_pll_bypass_sels[] = {"dram_pll", "dram_pll_ref_sel", }; +static const char * const gpu_pll_bypass_sels[] = {"gpu_pll", "gpu_pll_ref_sel", }; +static const char * const vpu_pll_bypass_sels[] = {"vpu_pll", "vpu_pll_ref_sel", }; +static const char * const arm_pll_bypass_sels[] = {"arm_pll", "arm_pll_ref_sel", }; +static const char * const sys_pll1_bypass_sels[] = {"sys_pll1", "sys_pll1_ref_sel", }; +static const char * const sys_pll2_bypass_sels[] = {"sys_pll2", "sys_pll2_ref_sel", }; +static const char * const sys_pll3_bypass_sels[] = {"sys_pll3", "sys_pll3_ref_sel", }; + +static const char * const imx8mp_a53_sels[] = {"osc_24m", "arm_pll_out", "sys_pll2_500m", + "sys_pll2_1000m", "sys_pll1_800m", "sys_pll1_400m", + "audio_pll1_out", "sys_pll3_out", }; + +static const char * const imx8mp_m7_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll2_250m", + "vpu_pll_out", "sys_pll1_800m", "audio_pll1_out", + "video_pll1_out", "sys_pll3_out", }; + +static const char * const imx8mp_ml_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_gpu3d_core_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_gpu3d_shader_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_gpu2d_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_audio_axi_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_hsio_axi_sels[] = {"osc_24m", "sys_pll2_500m", "sys_pll1_800m", + "sys_pll2_100m", "sys_pll2_200m", "clk_ext2", + "clk_ext4", "audio_pll2_out", }; + +static const char * const imx8mp_media_isp_sels[] = {"osc_24m", "sys_pll2_1000m", "sys_pll1_800m", + "sys_pll3_out", "sys_pll1_400m", "audio_pll2_out", + "clk_ext1", "sys_pll2_500m", }; + +static const char * const imx8mp_main_axi_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll1_800m", + "sys_pll2_250m", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "sys_pll1_100m",}; + +static const char * const imx8mp_enet_axi_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll1_800m", + "sys_pll2_250m", "sys_pll2_200m", "audio_pll1_out", + "video_pll1_out", "sys_pll3_out", }; + +static const char * const imx8mp_nand_usdhc_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll1_800m", + "sys_pll2_200m", "sys_pll1_133m", "sys_pll3_out", + "sys_pll2_250m", "audio_pll1_out", }; + +static const char * const imx8mp_vpu_bus_sels[] = {"osc_24m", "sys_pll1_800m", "vpu_pll_out", + "audio_pll2_out", "sys_pll3_out", "sys_pll2_1000m", + "sys_pll2_200m", "sys_pll1_100m", }; + +static const char * const imx8mp_media_axi_sels[] = {"osc_24m", "sys_pll2_1000m", "sys_pll1_800m", + "sys_pll3_out", "sys_pll1_40m", "audio_pll2_out", + "clk_ext1", "sys_pll2_500m", }; + +static const char * const imx8mp_media_apb_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll1_800m", + "sys_pll3_out", "sys_pll1_40m", "audio_pll2_out", + "clk_ext1", "sys_pll1_133m", }; + +static const char * const imx8mp_gpu_axi_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_gpu_ahb_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_noc_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_1000m", "sys_pll2_500m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_noc_io_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_1000m", "sys_pll2_500m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_ml_axi_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_ml_ahb_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out", + "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_ahb_sels[] = {"osc_24m", "sys_pll1_133m", "sys_pll1_800m", + "sys_pll1_400m", "sys_pll2_125m", "sys_pll3_out", + "audio_pll1_out", "video_pll1_out", }; + +static const char * const imx8mp_audio_ahb_sels[] = {"osc_24m", "sys_pll2_500m", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll2_166m", "sys_pll3_out", + "audio_pll1_out", "video_pll1_out", }; + +static const char * const imx8mp_mipi_dsi_esc_rx_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_80m", + "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll3_out", "clk_ext3", "audio_pll2_out", }; + +static const char * const imx8mp_dram_alt_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll1_100m", + "sys_pll2_500m", "sys_pll2_1000m", "sys_pll3_out", + "audio_pll1_out", "sys_pll1_266m", }; + +static const char * const imx8mp_dram_apb_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", + "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_250m", "audio_pll2_out", }; + +static const char * const imx8mp_vpu_g1_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll1_100m", "sys_pll2_125m", + "sys_pll3_out", "audio_pll1_out", }; + +static const char * const imx8mp_vpu_g2_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll1_100m", "sys_pll2_125m", + "sys_pll3_out", "audio_pll1_out", }; + +static const char * const imx8mp_can1_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", + "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_250m", "audio_pll2_out", }; + +static const char * const imx8mp_can2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", + "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_250m", "audio_pll2_out", }; + +static const char * const imx8mp_memrepair_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_pcie_phy_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll2_500m", + "clk_ext1", "clk_ext2", "clk_ext3", + "clk_ext4", "sys_pll1_400m", }; + +static const char * const imx8mp_pcie_aux_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll2_50m", + "sys_pll3_out", "sys_pll2_100m", "sys_pll1_80m", + "sys_pll1_160m", "sys_pll1_200m", }; + +static const char * const imx8mp_i2c5_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_i2c6_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_sai1_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", + "video_pll1_out", "sys_pll1_133m", "osc_hdmi", + "clk_ext1", "clk_ext2", }; + +static const char * const imx8mp_sai2_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", + "video_pll1_out", "sys_pll1_133m", "osc_hdmi", + "clk_ext2", "clk_ext3", }; + +static const char * const imx8mp_sai3_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", + "video_pll1_out", "sys_pll1_133m", "osc_hdmi", + "clk_ext3", "clk_ext4", }; + +static const char * const imx8mp_sai4_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", + "video_pll1_out", "sys_pll1_133m", "osc_hdmi", + "clk_ext1", "clk_ext2", }; + +static const char * const imx8mp_sai5_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", + "video_pll1_out", "sys_pll1_133m", "osc_hdmi", + "clk_ext2", "clk_ext3", }; + +static const char * const imx8mp_sai6_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", + "video_pll1_out", "sys_pll1_133m", "osc_hdmi", + "clk_ext3", "clk_ext4", }; + +static const char * const imx8mp_enet_qos_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll2_50m", + "sys_pll2_100m", "sys_pll1_160m", "audio_pll1_out", + "video_pll1_out", "clk_ext4", }; + +static const char * const imx8mp_enet_qos_timer_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_out", + "clk_ext1", "clk_ext2", "clk_ext3", + "clk_ext4", "video_pll1_out", }; + +static const char * const imx8mp_enet_ref_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll2_50m", + "sys_pll2_100m", "sys_pll1_160m", "audio_pll1_out", + "video_pll1_out", "clk_ext4", }; + +static const char * const imx8mp_enet_timer_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_out", + "clk_ext1", "clk_ext2", "clk_ext3", + "clk_ext4", "video_pll1_out", }; + +static const char * const imx8mp_enet_phy_ref_sels[] = {"osc_24m", "sys_pll2_50m", "sys_pll2_125m", + "sys_pll2_200m", "sys_pll2_500m", "audio_pll1_out", + "video_pll1_out", "audio_pll2_out", }; + +static const char * const imx8mp_nand_sels[] = {"osc_24m", "sys_pll2_500m", "audio_pll1_out", + "sys_pll1_400m", "audio_pll2_out", "sys_pll3_out", + "sys_pll2_250m", "video_pll1_out", }; + +static const char * const imx8mp_qspi_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll2_333m", + "sys_pll2_500m", "audio_pll2_out", "sys_pll1_266m", + "sys_pll3_out", "sys_pll1_100m", }; + +static const char * const imx8mp_usdhc1_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", + "sys_pll2_500m", "sys_pll3_out", "sys_pll1_266m", + "audio_pll2_out", "sys_pll1_100m", }; + +static const char * const imx8mp_usdhc2_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", + "sys_pll2_500m", "sys_pll3_out", "sys_pll1_266m", + "audio_pll2_out", "sys_pll1_100m", }; + +static const char * const imx8mp_i2c1_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_i2c2_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_i2c3_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_i2c4_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_uart1_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", + "sys_pll2_100m", "sys_pll3_out", "clk_ext2", + "clk_ext4", "audio_pll2_out", }; + +static const char * const imx8mp_uart2_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", + "sys_pll2_100m", "sys_pll3_out", "clk_ext2", + "clk_ext3", "audio_pll2_out", }; + +static const char * const imx8mp_uart3_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", + "sys_pll2_100m", "sys_pll3_out", "clk_ext2", + "clk_ext4", "audio_pll2_out", }; + +static const char * const imx8mp_uart4_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m", + "sys_pll2_100m", "sys_pll3_out", "clk_ext2", + "clk_ext3", "audio_pll2_out", }; + +static const char * const imx8mp_usb_core_ref_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m", + "sys_pll2_100m", "sys_pll2_200m", "clk_ext2", + "clk_ext3", "audio_pll2_out", }; + +static const char * const imx8mp_usb_phy_ref_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m", + "sys_pll2_100m", "sys_pll2_200m", "clk_ext2", + "clk_ext3", "audio_pll2_out", }; + +static const char * const imx8mp_gic_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", + "sys_pll2_100m", "sys_pll1_800m", + "sys_pll2_500m", "clk_ext4", "audio_pll2_out" }; + +static const char * const imx8mp_ecspi1_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", + "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_250m", "audio_pll2_out", }; + +static const char * const imx8mp_ecspi2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", + "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_250m", "audio_pll2_out", }; + +static const char * const imx8mp_pwm1_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", + "sys_pll1_40m", "sys_pll3_out", "clk_ext1", + "sys_pll1_80m", "video_pll1_out", }; + +static const char * const imx8mp_pwm2_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", + "sys_pll1_40m", "sys_pll3_out", "clk_ext1", + "sys_pll1_80m", "video_pll1_out", }; + +static const char * const imx8mp_pwm3_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", + "sys_pll1_40m", "sys_pll3_out", "clk_ext2", + "sys_pll1_80m", "video_pll1_out", }; + +static const char * const imx8mp_pwm4_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m", + "sys_pll1_40m", "sys_pll3_out", "clk_ext2", + "sys_pll1_80m", "video_pll1_out", }; + +static const char * const imx8mp_gpt1_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_400m", + "sys_pll1_40m", "video_pll1_out", "sys_pll1_80m", + "audio_pll1_out", "clk_ext1" }; + +static const char * const imx8mp_gpt2_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_400m", + "sys_pll1_40m", "video_pll1_out", "sys_pll1_80m", + "audio_pll1_out", "clk_ext2" }; + +static const char * const imx8mp_gpt3_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_400m", + "sys_pll1_40m", "video_pll1_out", "sys_pll1_80m", + "audio_pll1_out", "clk_ext3" }; + +static const char * const imx8mp_gpt4_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_400m", + "sys_pll1_40m", "video_pll1_out", "sys_pll1_80m", + "audio_pll1_out", "clk_ext1" }; + +static const char * const imx8mp_gpt5_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_400m", + "sys_pll1_40m", "video_pll1_out", "sys_pll1_80m", + "audio_pll1_out", "clk_ext2" }; + +static const char * const imx8mp_gpt6_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_400m", + "sys_pll1_40m", "video_pll1_out", "sys_pll1_80m", + "audio_pll1_out", "clk_ext3" }; + +static const char * const imx8mp_wdog_sels[] = {"osc_24m", "sys_pll1_133m", "sys_pll1_160m", + "vpu_pll_out", "sys_pll2_125m", "sys_pll3_out", + "sys_pll1_80m", "sys_pll2_166m" }; + +static const char * const imx8mp_wrclk_sels[] = {"osc_24m", "sys_pll1_40m", "vpu_pll_out", + "sys_pll3_out", "sys_pll2_200m", "sys_pll1_266m", + "sys_pll2_500m", "sys_pll1_100m" }; + +static const char * const imx8mp_ipp_do_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll1_133m", + "sys_pll1_200m", "audio_pll2_out", "sys_pll2_500m", + "vpu_pll_out", "sys_pll1_80m" }; + +static const char * const imx8mp_ipp_do_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_400m", + "sys_pll1_166m", "sys_pll3_out", "audio_pll1_out", + "video_pll1_out", "osc_32k" }; + +static const char * const imx8mp_hdmi_fdcc_tst_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", + "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out", + "audio_pll2_out", "video_pll1_out", }; + +static const char * const imx8mp_hdmi_27m_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m", + "sys_pll3_out", "audio_pll1_out", "video_pll1_out", + "audio_pll2_out", "sys_pll1_133m", }; + +static const char * const imx8mp_hdmi_ref_266m_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll3_out", + "sys_pll2_333m", "sys_pll1_266m", "sys_pll2_200m", + "audio_pll1_out", "video_pll1_out", }; + +static const char * const imx8mp_usdhc3_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", + "sys_pll2_500m", "sys_pll3_out", "sys_pll1_266m", + "audio_pll2_out", "sys_pll1_100m", }; + +static const char * const imx8mp_media_cam1_pix_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", + "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll3_out", "audio_pll2_out", + "video_pll1_out", }; + +static const char * const imx8mp_media_mipi_phy1_ref_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m", + "sys_pll1_800m", "sys_pll2_1000m", + "clk_ext2", "audio_pll2_out", + "video_pll1_out", }; + +static const char * const imx8mp_media_disp1_pix_sels[] = {"osc_24m", "video_pll1_out", "audio_pll2_out", + "audio_pll1_out", "sys_pll1_800m", + "sys_pll2_1000m", "sys_pll3_out", "clk_ext4", }; + +static const char * const imx8mp_media_cam2_pix_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", + "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll3_out", "audio_pll2_out", + "video_pll1_out", }; + +static const char * const imx8mp_media_mipi_phy2_ref_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m", + "sys_pll1_800m", "sys_pll2_1000m", + "clk_ext2", "audio_pll2_out", + "video_pll1_out", }; + +static const char * const imx8mp_media_mipi_csi2_esc_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_80m", + "sys_pll1_800m", "sys_pll2_1000m", + "sys_pll3_out", "clk_ext3", + "audio_pll2_out", }; + +static const char * const imx8mp_pcie2_ctrl_sels[] = {"osc_24m", "sys_pll2_250m", "sys_pll2_200m", + "sys_pll1_266m", "sys_pll1_800m", "sys_pll2_500m", + "sys_pll2_333m", "sys_pll3_out", }; + +static const char * const imx8mp_pcie2_phy_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll2_500m", + "clk_ext1", "clk_ext2", "clk_ext3", + "clk_ext4", "sys_pll1_400m", }; + +static const char * const imx8mp_media_mipi_test_byte_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll2_50m", + "sys_pll3_out", "sys_pll2_100m", + "sys_pll1_80m", "sys_pll1_160m", + "sys_pll1_200m", }; + +static const char * const imx8mp_ecspi3_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", + "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", + "sys_pll2_250m", "audio_pll2_out", }; + +static const char * const imx8mp_pdm_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_out", + "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out", + "clk_ext3", "audio_pll2_out", }; + +static const char * const imx8mp_vpu_vc8000e_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", + "sys_pll2_1000m", "audio_pll2_out", "sys_pll2_125m", + "sys_pll3_out", "audio_pll1_out", }; + +static const char * const imx8mp_sai7_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out", + "video_pll1_out", "sys_pll1_133m", "osc_hdmi", + "clk_ext3", "clk_ext4", }; + +static const char * const imx8mp_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", }; + +static struct clk_hw **hws; +static struct clk_hw_onecell_data *clk_hw_data; + +static const int uart_clk_ids[] = { + IMX8MP_CLK_UART1_ROOT, + IMX8MP_CLK_UART2_ROOT, + IMX8MP_CLK_UART3_ROOT, + IMX8MP_CLK_UART4_ROOT, +}; +static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1]; + +static int imx8mp_clocks_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + void __iomem *anatop_base, *ccm_base; + int i; + + np = of_find_compatible_node(NULL, NULL, "fsl,imx8mp-anatop"); + anatop_base = of_iomap(np, 0); + if (WARN_ON(!anatop_base)) + return -ENOMEM; + + np = dev->of_node; + ccm_base = devm_platform_ioremap_resource(pdev, 0); + if (WARN_ON(IS_ERR(ccm_base))) { + iounmap(anatop_base); + return PTR_ERR(ccm_base); + } + + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, IMX8MP_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) { + iounmap(anatop_base); + return -ENOMEM; + } + + clk_hw_data->num = IMX8MP_CLK_END; + hws = clk_hw_data->hws; + + hws[IMX8MP_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); + hws[IMX8MP_CLK_24M] = imx_obtain_fixed_clk_hw(np, "osc_24m"); + hws[IMX8MP_CLK_32K] = imx_obtain_fixed_clk_hw(np, "osc_32k"); + hws[IMX8MP_CLK_EXT1] = imx_obtain_fixed_clk_hw(np, "clk_ext1"); + hws[IMX8MP_CLK_EXT2] = imx_obtain_fixed_clk_hw(np, "clk_ext2"); + hws[IMX8MP_CLK_EXT3] = imx_obtain_fixed_clk_hw(np, "clk_ext3"); + hws[IMX8MP_CLK_EXT4] = imx_obtain_fixed_clk_hw(np, "clk_ext4"); + + hws[IMX8MP_AUDIO_PLL1_REF_SEL] = imx_clk_hw_mux("audio_pll1_ref_sel", anatop_base + 0x0, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_AUDIO_PLL2_REF_SEL] = imx_clk_hw_mux("audio_pll2_ref_sel", anatop_base + 0x14, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_VIDEO_PLL1_REF_SEL] = imx_clk_hw_mux("video_pll1_ref_sel", anatop_base + 0x28, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_DRAM_PLL_REF_SEL] = imx_clk_hw_mux("dram_pll_ref_sel", anatop_base + 0x50, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_GPU_PLL_REF_SEL] = imx_clk_hw_mux("gpu_pll_ref_sel", anatop_base + 0x64, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_VPU_PLL_REF_SEL] = imx_clk_hw_mux("vpu_pll_ref_sel", anatop_base + 0x74, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_ARM_PLL_REF_SEL] = imx_clk_hw_mux("arm_pll_ref_sel", anatop_base + 0x84, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_SYS_PLL1_REF_SEL] = imx_clk_hw_mux("sys_pll1_ref_sel", anatop_base + 0x94, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_SYS_PLL2_REF_SEL] = imx_clk_hw_mux("sys_pll2_ref_sel", anatop_base + 0x104, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MP_SYS_PLL3_REF_SEL] = imx_clk_hw_mux("sys_pll3_ref_sel", anatop_base + 0x114, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + + hws[IMX8MP_AUDIO_PLL1] = imx_clk_hw_pll14xx("audio_pll1", "audio_pll1_ref_sel", anatop_base, &imx_1443x_pll); + hws[IMX8MP_AUDIO_PLL2] = imx_clk_hw_pll14xx("audio_pll2", "audio_pll2_ref_sel", anatop_base + 0x14, &imx_1443x_pll); + hws[IMX8MP_VIDEO_PLL1] = imx_clk_hw_pll14xx("video_pll1", "video_pll1_ref_sel", anatop_base + 0x28, &imx_1443x_pll); + hws[IMX8MP_DRAM_PLL] = imx_clk_hw_pll14xx("dram_pll", "dram_pll_ref_sel", anatop_base + 0x50, &imx_1443x_dram_pll); + hws[IMX8MP_GPU_PLL] = imx_clk_hw_pll14xx("gpu_pll", "gpu_pll_ref_sel", anatop_base + 0x64, &imx_1416x_pll); + hws[IMX8MP_VPU_PLL] = imx_clk_hw_pll14xx("vpu_pll", "vpu_pll_ref_sel", anatop_base + 0x74, &imx_1416x_pll); + hws[IMX8MP_ARM_PLL] = imx_clk_hw_pll14xx("arm_pll", "arm_pll_ref_sel", anatop_base + 0x84, &imx_1416x_pll); + hws[IMX8MP_SYS_PLL1] = imx_clk_hw_pll14xx("sys_pll1", "sys_pll1_ref_sel", anatop_base + 0x94, &imx_1416x_pll); + hws[IMX8MP_SYS_PLL2] = imx_clk_hw_pll14xx("sys_pll2", "sys_pll2_ref_sel", anatop_base + 0x104, &imx_1416x_pll); + hws[IMX8MP_SYS_PLL3] = imx_clk_hw_pll14xx("sys_pll3", "sys_pll3_ref_sel", anatop_base + 0x114, &imx_1416x_pll); + + hws[IMX8MP_AUDIO_PLL1_BYPASS] = imx_clk_hw_mux_flags("audio_pll1_bypass", anatop_base, 4, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_AUDIO_PLL2_BYPASS] = imx_clk_hw_mux_flags("audio_pll2_bypass", anatop_base + 0x14, 4, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_VIDEO_PLL1_BYPASS] = imx_clk_hw_mux_flags("video_pll1_bypass", anatop_base + 0x28, 4, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_DRAM_PLL_BYPASS] = imx_clk_hw_mux_flags("dram_pll_bypass", anatop_base + 0x50, 4, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_GPU_PLL_BYPASS] = imx_clk_hw_mux_flags("gpu_pll_bypass", anatop_base + 0x64, 4, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_VPU_PLL_BYPASS] = imx_clk_hw_mux_flags("vpu_pll_bypass", anatop_base + 0x74, 4, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_ARM_PLL_BYPASS] = imx_clk_hw_mux_flags("arm_pll_bypass", anatop_base + 0x84, 4, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_SYS_PLL1_BYPASS] = imx_clk_hw_mux_flags("sys_pll1_bypass", anatop_base + 0x94, 4, 1, sys_pll1_bypass_sels, ARRAY_SIZE(sys_pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_SYS_PLL2_BYPASS] = imx_clk_hw_mux_flags("sys_pll2_bypass", anatop_base + 0x104, 4, 1, sys_pll2_bypass_sels, ARRAY_SIZE(sys_pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MP_SYS_PLL3_BYPASS] = imx_clk_hw_mux_flags("sys_pll3_bypass", anatop_base + 0x114, 4, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT); + + hws[IMX8MP_AUDIO_PLL1_OUT] = imx_clk_hw_gate("audio_pll1_out", "audio_pll1_bypass", anatop_base, 13); + hws[IMX8MP_AUDIO_PLL2_OUT] = imx_clk_hw_gate("audio_pll2_out", "audio_pll2_bypass", anatop_base + 0x14, 13); + hws[IMX8MP_VIDEO_PLL1_OUT] = imx_clk_hw_gate("video_pll1_out", "video_pll1_bypass", anatop_base + 0x28, 13); + hws[IMX8MP_DRAM_PLL_OUT] = imx_clk_hw_gate("dram_pll_out", "dram_pll_bypass", anatop_base + 0x50, 13); + hws[IMX8MP_GPU_PLL_OUT] = imx_clk_hw_gate("gpu_pll_out", "gpu_pll_bypass", anatop_base + 0x64, 11); + hws[IMX8MP_VPU_PLL_OUT] = imx_clk_hw_gate("vpu_pll_out", "vpu_pll_bypass", anatop_base + 0x74, 11); + hws[IMX8MP_ARM_PLL_OUT] = imx_clk_hw_gate("arm_pll_out", "arm_pll_bypass", anatop_base + 0x84, 11); + hws[IMX8MP_SYS_PLL1_OUT] = imx_clk_hw_gate("sys_pll1_out", "sys_pll1_bypass", anatop_base + 0x94, 11); + hws[IMX8MP_SYS_PLL2_OUT] = imx_clk_hw_gate("sys_pll2_out", "sys_pll2_bypass", anatop_base + 0x104, 11); + hws[IMX8MP_SYS_PLL3_OUT] = imx_clk_hw_gate("sys_pll3_out", "sys_pll3_bypass", anatop_base + 0x114, 11); + + hws[IMX8MP_SYS_PLL1_40M] = imx_clk_hw_fixed_factor("sys_pll1_40m", "sys_pll1_out", 1, 20); + hws[IMX8MP_SYS_PLL1_80M] = imx_clk_hw_fixed_factor("sys_pll1_80m", "sys_pll1_out", 1, 10); + hws[IMX8MP_SYS_PLL1_100M] = imx_clk_hw_fixed_factor("sys_pll1_100m", "sys_pll1_out", 1, 8); + hws[IMX8MP_SYS_PLL1_133M] = imx_clk_hw_fixed_factor("sys_pll1_133m", "sys_pll1_out", 1, 6); + hws[IMX8MP_SYS_PLL1_160M] = imx_clk_hw_fixed_factor("sys_pll1_160m", "sys_pll1_out", 1, 5); + hws[IMX8MP_SYS_PLL1_200M] = imx_clk_hw_fixed_factor("sys_pll1_200m", "sys_pll1_out", 1, 4); + hws[IMX8MP_SYS_PLL1_266M] = imx_clk_hw_fixed_factor("sys_pll1_266m", "sys_pll1_out", 1, 3); + hws[IMX8MP_SYS_PLL1_400M] = imx_clk_hw_fixed_factor("sys_pll1_400m", "sys_pll1_out", 1, 2); + hws[IMX8MP_SYS_PLL1_800M] = imx_clk_hw_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1); + + hws[IMX8MP_SYS_PLL2_50M] = imx_clk_hw_fixed_factor("sys_pll2_50m", "sys_pll2_out", 1, 20); + hws[IMX8MP_SYS_PLL2_100M] = imx_clk_hw_fixed_factor("sys_pll2_100m", "sys_pll2_out", 1, 10); + hws[IMX8MP_SYS_PLL2_125M] = imx_clk_hw_fixed_factor("sys_pll2_125m", "sys_pll2_out", 1, 8); + hws[IMX8MP_SYS_PLL2_166M] = imx_clk_hw_fixed_factor("sys_pll2_166m", "sys_pll2_out", 1, 6); + hws[IMX8MP_SYS_PLL2_200M] = imx_clk_hw_fixed_factor("sys_pll2_200m", "sys_pll2_out", 1, 5); + hws[IMX8MP_SYS_PLL2_250M] = imx_clk_hw_fixed_factor("sys_pll2_250m", "sys_pll2_out", 1, 4); + hws[IMX8MP_SYS_PLL2_333M] = imx_clk_hw_fixed_factor("sys_pll2_333m", "sys_pll2_out", 1, 3); + hws[IMX8MP_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_out", 1, 2); + hws[IMX8MP_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + + hws[IMX8MP_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", ccm_base + 0x8000, 24, 3, imx8mp_a53_sels, ARRAY_SIZE(imx8mp_a53_sels)); + hws[IMX8MP_CLK_M7_SRC] = imx_clk_hw_mux2("arm_m7_src", ccm_base + 0x8080, 24, 3, imx8mp_m7_sels, ARRAY_SIZE(imx8mp_m7_sels)); + hws[IMX8MP_CLK_ML_SRC] = imx_clk_hw_mux2("ml_src", ccm_base + 0x8100, 24, 3, imx8mp_ml_sels, ARRAY_SIZE(imx8mp_ml_sels)); + hws[IMX8MP_CLK_GPU3D_CORE_SRC] = imx_clk_hw_mux2("gpu3d_core_src", ccm_base + 0x8180, 24, 3, imx8mp_gpu3d_core_sels, ARRAY_SIZE(imx8mp_gpu3d_core_sels)); + hws[IMX8MP_CLK_GPU3D_SHADER_SRC] = imx_clk_hw_mux2("gpu3d_shader_src", ccm_base + 0x8200, 24, 3, imx8mp_gpu3d_shader_sels, ARRAY_SIZE(imx8mp_gpu3d_shader_sels)); + hws[IMX8MP_CLK_GPU2D_SRC] = imx_clk_hw_mux2("gpu2d_src", ccm_base + 0x8280, 24, 3, imx8mp_gpu2d_sels, ARRAY_SIZE(imx8mp_gpu2d_sels)); + hws[IMX8MP_CLK_AUDIO_AXI_SRC] = imx_clk_hw_mux2("audio_axi_src", ccm_base + 0x8300, 24, 3, imx8mp_audio_axi_sels, ARRAY_SIZE(imx8mp_audio_axi_sels)); + hws[IMX8MP_CLK_HSIO_AXI_SRC] = imx_clk_hw_mux2("hsio_axi_src", ccm_base + 0x8380, 24, 3, imx8mp_hsio_axi_sels, ARRAY_SIZE(imx8mp_hsio_axi_sels)); + hws[IMX8MP_CLK_MEDIA_ISP_SRC] = imx_clk_hw_mux2("media_isp_src", ccm_base + 0x8400, 24, 3, imx8mp_media_isp_sels, ARRAY_SIZE(imx8mp_media_isp_sels)); + hws[IMX8MP_CLK_A53_CG] = imx_clk_hw_gate3("arm_a53_cg", "arm_a53_src", ccm_base + 0x8000, 28); + hws[IMX8MP_CLK_M4_CG] = imx_clk_hw_gate3("arm_m7_cg", "arm_m7_src", ccm_base + 0x8080, 28); + hws[IMX8MP_CLK_ML_CG] = imx_clk_hw_gate3("ml_cg", "ml_src", ccm_base + 0x8100, 28); + hws[IMX8MP_CLK_GPU3D_CORE_CG] = imx_clk_hw_gate3("gpu3d_core_cg", "gpu3d_core_src", ccm_base + 0x8180, 28); + hws[IMX8MP_CLK_GPU3D_SHADER_CG] = imx_clk_hw_gate3("gpu3d_shader_cg", "gpu3d_shader_src", ccm_base + 0x8200, 28); + hws[IMX8MP_CLK_GPU2D_CG] = imx_clk_hw_gate3("gpu2d_cg", "gpu2d_src", ccm_base + 0x8280, 28); + hws[IMX8MP_CLK_AUDIO_AXI_CG] = imx_clk_hw_gate3("audio_axi_cg", "audio_axi_src", ccm_base + 0x8300, 28); + hws[IMX8MP_CLK_HSIO_AXI_CG] = imx_clk_hw_gate3("hsio_axi_cg", "hsio_axi_src", ccm_base + 0x8380, 28); + hws[IMX8MP_CLK_MEDIA_ISP_CG] = imx_clk_hw_gate3("media_isp_cg", "media_isp_src", ccm_base + 0x8400, 28); + hws[IMX8MP_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", ccm_base + 0x8000, 0, 3); + hws[IMX8MP_CLK_M7_DIV] = imx_clk_hw_divider2("arm_m7_div", "arm_m7_cg", ccm_base + 0x8080, 0, 3); + hws[IMX8MP_CLK_ML_DIV] = imx_clk_hw_divider2("ml_div", "ml_cg", ccm_base + 0x8100, 0, 3); + hws[IMX8MP_CLK_GPU3D_CORE_DIV] = imx_clk_hw_divider2("gpu3d_core_div", "gpu3d_core_cg", ccm_base + 0x8180, 0, 3); + hws[IMX8MP_CLK_GPU3D_SHADER_DIV] = imx_clk_hw_divider2("gpu3d_shader_div", "gpu3d_shader_cg", ccm_base + 0x8200, 0, 3); + hws[IMX8MP_CLK_GPU2D_DIV] = imx_clk_hw_divider2("gpu2d_div", "gpu2d_cg", ccm_base + 0x8280, 0, 3); + hws[IMX8MP_CLK_AUDIO_AXI_DIV] = imx_clk_hw_divider2("audio_axi_div", "audio_axi_cg", ccm_base + 0x8300, 0, 3); + hws[IMX8MP_CLK_HSIO_AXI_DIV] = imx_clk_hw_divider2("hsio_axi_div", "hsio_axi_cg", ccm_base + 0x8380, 0, 3); + hws[IMX8MP_CLK_MEDIA_ISP_DIV] = imx_clk_hw_divider2("media_isp_div", "media_isp_cg", ccm_base + 0x8400, 0, 3); + + hws[IMX8MP_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi", imx8mp_main_axi_sels, ccm_base + 0x8800); + hws[IMX8MP_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mp_enet_axi_sels, ccm_base + 0x8880); + hws[IMX8MP_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite_critical("nand_usdhc_bus", imx8mp_nand_usdhc_sels, ccm_base + 0x8900); + hws[IMX8MP_CLK_VPU_BUS] = imx8m_clk_hw_composite("vpu_bus", imx8mp_vpu_bus_sels, ccm_base + 0x8980); + hws[IMX8MP_CLK_MEDIA_AXI] = imx8m_clk_hw_composite("media_axi", imx8mp_media_axi_sels, ccm_base + 0x8a00); + hws[IMX8MP_CLK_MEDIA_APB] = imx8m_clk_hw_composite("media_apb", imx8mp_media_apb_sels, ccm_base + 0x8a80); + hws[IMX8MP_CLK_HDMI_APB] = imx8m_clk_hw_composite("hdmi_apb", imx8mp_media_apb_sels, ccm_base + 0x8b00); + hws[IMX8MP_CLK_HDMI_AXI] = imx8m_clk_hw_composite("hdmi_axi", imx8mp_media_apb_sels, ccm_base + 0x8b80); + hws[IMX8MP_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mp_gpu_axi_sels, ccm_base + 0x8c00); + hws[IMX8MP_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mp_gpu_ahb_sels, ccm_base + 0x8c80); + hws[IMX8MP_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mp_noc_sels, ccm_base + 0x8d00); + hws[IMX8MP_CLK_NOC_IO] = imx8m_clk_hw_composite_critical("noc_io", imx8mp_noc_io_sels, ccm_base + 0x8d80); + hws[IMX8MP_CLK_ML_AXI] = imx8m_clk_hw_composite("ml_axi", imx8mp_ml_axi_sels, ccm_base + 0x8e00); + hws[IMX8MP_CLK_ML_AHB] = imx8m_clk_hw_composite("ml_ahb", imx8mp_ml_ahb_sels, ccm_base + 0x8e80); + + hws[IMX8MP_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb_root", imx8mp_ahb_sels, ccm_base + 0x9000); + hws[IMX8MP_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mp_audio_ahb_sels, ccm_base + 0x9100); + hws[IMX8MP_CLK_MIPI_DSI_ESC_RX] = imx8m_clk_hw_composite("mipi_dsi_esc_rx", imx8mp_mipi_dsi_esc_rx_sels, ccm_base + 0x9200); + + hws[IMX8MP_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb_root", ccm_base + 0x9080, 0, 1); + hws[IMX8MP_CLK_IPG_AUDIO_ROOT] = imx_clk_hw_divider2("ipg_audio_root", "audio_ahb", ccm_base + 0x9180, 0, 1); + + hws[IMX8MP_CLK_DRAM_ALT] = imx8m_clk_hw_composite("dram_alt", imx8mp_dram_alt_sels, ccm_base + 0xa000); + hws[IMX8MP_CLK_DRAM_APB] = imx8m_clk_hw_composite_critical("dram_apb", imx8mp_dram_apb_sels, ccm_base + 0xa080); + hws[IMX8MP_CLK_VPU_G1] = imx8m_clk_hw_composite("vpu_g1", imx8mp_vpu_g1_sels, ccm_base + 0xa100); + hws[IMX8MP_CLK_VPU_G2] = imx8m_clk_hw_composite("vpu_g2", imx8mp_vpu_g2_sels, ccm_base + 0xa180); + hws[IMX8MP_CLK_CAN1] = imx8m_clk_hw_composite("can1", imx8mp_can1_sels, ccm_base + 0xa200); + hws[IMX8MP_CLK_CAN2] = imx8m_clk_hw_composite("can2", imx8mp_can2_sels, ccm_base + 0xa280); + hws[IMX8MP_CLK_MEMREPAIR] = imx8m_clk_hw_composite("memrepair", imx8mp_memrepair_sels, ccm_base + 0xa300); + hws[IMX8MP_CLK_PCIE_PHY] = imx8m_clk_hw_composite("pcie_phy", imx8mp_pcie_phy_sels, ccm_base + 0xa380); + hws[IMX8MP_CLK_PCIE_AUX] = imx8m_clk_hw_composite("pcie_aux", imx8mp_pcie_aux_sels, ccm_base + 0xa400); + hws[IMX8MP_CLK_I2C5] = imx8m_clk_hw_composite("i2c5", imx8mp_i2c5_sels, ccm_base + 0xa480); + hws[IMX8MP_CLK_I2C6] = imx8m_clk_hw_composite("i2c6", imx8mp_i2c6_sels, ccm_base + 0xa500); + hws[IMX8MP_CLK_SAI1] = imx8m_clk_hw_composite("sai1", imx8mp_sai1_sels, ccm_base + 0xa580); + hws[IMX8MP_CLK_SAI2] = imx8m_clk_hw_composite("sai2", imx8mp_sai2_sels, ccm_base + 0xa600); + hws[IMX8MP_CLK_SAI3] = imx8m_clk_hw_composite("sai3", imx8mp_sai3_sels, ccm_base + 0xa680); + hws[IMX8MP_CLK_SAI4] = imx8m_clk_hw_composite("sai4", imx8mp_sai4_sels, ccm_base + 0xa700); + hws[IMX8MP_CLK_SAI5] = imx8m_clk_hw_composite("sai5", imx8mp_sai5_sels, ccm_base + 0xa780); + hws[IMX8MP_CLK_SAI6] = imx8m_clk_hw_composite("sai6", imx8mp_sai6_sels, ccm_base + 0xa800); + hws[IMX8MP_CLK_ENET_QOS] = imx8m_clk_hw_composite("enet_qos", imx8mp_enet_qos_sels, ccm_base + 0xa880); + hws[IMX8MP_CLK_ENET_QOS_TIMER] = imx8m_clk_hw_composite("enet_qos_timer", imx8mp_enet_qos_timer_sels, ccm_base + 0xa900); + hws[IMX8MP_CLK_ENET_REF] = imx8m_clk_hw_composite("enet_ref", imx8mp_enet_ref_sels, ccm_base + 0xa980); + hws[IMX8MP_CLK_ENET_TIMER] = imx8m_clk_hw_composite("enet_timer", imx8mp_enet_timer_sels, ccm_base + 0xaa00); + hws[IMX8MP_CLK_ENET_PHY_REF] = imx8m_clk_hw_composite("enet_phy_ref", imx8mp_enet_phy_ref_sels, ccm_base + 0xaa80); + hws[IMX8MP_CLK_NAND] = imx8m_clk_hw_composite("nand", imx8mp_nand_sels, ccm_base + 0xab00); + hws[IMX8MP_CLK_QSPI] = imx8m_clk_hw_composite("qspi", imx8mp_qspi_sels, ccm_base + 0xab80); + hws[IMX8MP_CLK_USDHC1] = imx8m_clk_hw_composite("usdhc1", imx8mp_usdhc1_sels, ccm_base + 0xac00); + hws[IMX8MP_CLK_USDHC2] = imx8m_clk_hw_composite("usdhc2", imx8mp_usdhc2_sels, ccm_base + 0xac80); + hws[IMX8MP_CLK_I2C1] = imx8m_clk_hw_composite("i2c1", imx8mp_i2c1_sels, ccm_base + 0xad00); + hws[IMX8MP_CLK_I2C2] = imx8m_clk_hw_composite("i2c2", imx8mp_i2c2_sels, ccm_base + 0xad80); + hws[IMX8MP_CLK_I2C3] = imx8m_clk_hw_composite("i2c3", imx8mp_i2c3_sels, ccm_base + 0xae00); + hws[IMX8MP_CLK_I2C4] = imx8m_clk_hw_composite("i2c4", imx8mp_i2c4_sels, ccm_base + 0xae80); + + hws[IMX8MP_CLK_UART1] = imx8m_clk_hw_composite("uart1", imx8mp_uart1_sels, ccm_base + 0xaf00); + hws[IMX8MP_CLK_UART2] = imx8m_clk_hw_composite("uart2", imx8mp_uart2_sels, ccm_base + 0xaf80); + hws[IMX8MP_CLK_UART3] = imx8m_clk_hw_composite("uart3", imx8mp_uart3_sels, ccm_base + 0xb000); + hws[IMX8MP_CLK_UART4] = imx8m_clk_hw_composite("uart4", imx8mp_uart4_sels, ccm_base + 0xb080); + hws[IMX8MP_CLK_USB_CORE_REF] = imx8m_clk_hw_composite("usb_core_ref", imx8mp_usb_core_ref_sels, ccm_base + 0xb100); + hws[IMX8MP_CLK_USB_PHY_REF] = imx8m_clk_hw_composite("usb_phy_ref", imx8mp_usb_phy_ref_sels, ccm_base + 0xb180); + hws[IMX8MP_CLK_GIC] = imx8m_clk_hw_composite_critical("gic", imx8mp_gic_sels, ccm_base + 0xb200); + hws[IMX8MP_CLK_ECSPI1] = imx8m_clk_hw_composite("ecspi1", imx8mp_ecspi1_sels, ccm_base + 0xb280); + hws[IMX8MP_CLK_ECSPI2] = imx8m_clk_hw_composite("ecspi2", imx8mp_ecspi2_sels, ccm_base + 0xb300); + hws[IMX8MP_CLK_PWM1] = imx8m_clk_hw_composite("pwm1", imx8mp_pwm1_sels, ccm_base + 0xb380); + hws[IMX8MP_CLK_PWM2] = imx8m_clk_hw_composite("pwm2", imx8mp_pwm2_sels, ccm_base + 0xb400); + hws[IMX8MP_CLK_PWM3] = imx8m_clk_hw_composite("pwm3", imx8mp_pwm3_sels, ccm_base + 0xb480); + hws[IMX8MP_CLK_PWM4] = imx8m_clk_hw_composite("pwm4", imx8mp_pwm4_sels, ccm_base + 0xb500); + + hws[IMX8MP_CLK_GPT1] = imx8m_clk_hw_composite("gpt1", imx8mp_gpt1_sels, ccm_base + 0xb580); + hws[IMX8MP_CLK_GPT2] = imx8m_clk_hw_composite("gpt2", imx8mp_gpt2_sels, ccm_base + 0xb600); + hws[IMX8MP_CLK_GPT3] = imx8m_clk_hw_composite("gpt3", imx8mp_gpt3_sels, ccm_base + 0xb680); + hws[IMX8MP_CLK_GPT4] = imx8m_clk_hw_composite("gpt4", imx8mp_gpt4_sels, ccm_base + 0xb700); + hws[IMX8MP_CLK_GPT5] = imx8m_clk_hw_composite("gpt5", imx8mp_gpt5_sels, ccm_base + 0xb780); + hws[IMX8MP_CLK_GPT6] = imx8m_clk_hw_composite("gpt6", imx8mp_gpt6_sels, ccm_base + 0xb800); + hws[IMX8MP_CLK_WDOG] = imx8m_clk_hw_composite("wdog", imx8mp_wdog_sels, ccm_base + 0xb900); + hws[IMX8MP_CLK_WRCLK] = imx8m_clk_hw_composite("wrclk", imx8mp_wrclk_sels, ccm_base + 0xb980); + hws[IMX8MP_CLK_IPP_DO_CLKO1] = imx8m_clk_hw_composite("ipp_do_clko1", imx8mp_ipp_do_clko1_sels, ccm_base + 0xba00); + hws[IMX8MP_CLK_IPP_DO_CLKO2] = imx8m_clk_hw_composite("ipp_do_clko2", imx8mp_ipp_do_clko2_sels, ccm_base + 0xba80); + hws[IMX8MP_CLK_HDMI_FDCC_TST] = imx8m_clk_hw_composite("hdmi_fdcc_tst", imx8mp_hdmi_fdcc_tst_sels, ccm_base + 0xbb00); + hws[IMX8MP_CLK_HDMI_27M] = imx8m_clk_hw_composite("hdmi_27m", imx8mp_hdmi_27m_sels, ccm_base + 0xbb80); + hws[IMX8MP_CLK_HDMI_REF_266M] = imx8m_clk_hw_composite("hdmi_ref_266m", imx8mp_hdmi_ref_266m_sels, ccm_base + 0xbc00); + hws[IMX8MP_CLK_USDHC3] = imx8m_clk_hw_composite("usdhc3", imx8mp_usdhc3_sels, ccm_base + 0xbc80); + hws[IMX8MP_CLK_MEDIA_CAM1_PIX] = imx8m_clk_hw_composite("media_cam1_pix", imx8mp_media_cam1_pix_sels, ccm_base + 0xbd00); + hws[IMX8MP_CLK_MEDIA_MIPI_PHY1_REF] = imx8m_clk_hw_composite("media_mipi_phy1_ref", imx8mp_media_mipi_phy1_ref_sels, ccm_base + 0xbd80); + hws[IMX8MP_CLK_MEDIA_DISP1_PIX] = imx8m_clk_hw_composite("media_disp1_pix", imx8mp_media_disp1_pix_sels, ccm_base + 0xbe00); + hws[IMX8MP_CLK_MEDIA_CAM2_PIX] = imx8m_clk_hw_composite("media_cam2_pix", imx8mp_media_cam2_pix_sels, ccm_base + 0xbe80); + hws[IMX8MP_CLK_MEDIA_MIPI_PHY2_REF] = imx8m_clk_hw_composite("media_mipi_phy2_ref", imx8mp_media_mipi_phy2_ref_sels, ccm_base + 0xbf00); + hws[IMX8MP_CLK_MEDIA_MIPI_CSI2_ESC] = imx8m_clk_hw_composite("media_mipi_csi2_esc", imx8mp_media_mipi_csi2_esc_sels, ccm_base + 0xbf80); + hws[IMX8MP_CLK_PCIE2_CTRL] = imx8m_clk_hw_composite("pcie2_ctrl", imx8mp_pcie2_ctrl_sels, ccm_base + 0xc000); + hws[IMX8MP_CLK_PCIE2_PHY] = imx8m_clk_hw_composite("pcie2_phy", imx8mp_pcie2_phy_sels, ccm_base + 0xc080); + hws[IMX8MP_CLK_MEDIA_MIPI_TEST_BYTE] = imx8m_clk_hw_composite("media_mipi_test_byte", imx8mp_media_mipi_test_byte_sels, ccm_base + 0xc100); + hws[IMX8MP_CLK_ECSPI3] = imx8m_clk_hw_composite("ecspi3", imx8mp_ecspi3_sels, ccm_base + 0xc180); + hws[IMX8MP_CLK_PDM] = imx8m_clk_hw_composite("pdm", imx8mp_pdm_sels, ccm_base + 0xc200); + hws[IMX8MP_CLK_VPU_VC8000E] = imx8m_clk_hw_composite("vpu_vc8000e", imx8mp_vpu_vc8000e_sels, ccm_base + 0xc280); + hws[IMX8MP_CLK_SAI7] = imx8m_clk_hw_composite("sai7", imx8mp_sai7_sels, ccm_base + 0xc300); + + hws[IMX8MP_CLK_DRAM_ALT_ROOT] = imx_clk_hw_fixed_factor("dram_alt_root", "dram_alt", 1, 4); + hws[IMX8MP_CLK_DRAM_CORE] = imx_clk_hw_mux2_flags("dram_core_clk", ccm_base + 0x9800, 24, 1, imx8mp_dram_core_sels, ARRAY_SIZE(imx8mp_dram_core_sels), CLK_IS_CRITICAL); + + hws[IMX8MP_CLK_DRAM1_ROOT] = imx_clk_hw_gate4_flags("dram1_root_clk", "dram_core_clk", ccm_base + 0x4050, 0, CLK_IS_CRITICAL); + hws[IMX8MP_CLK_ECSPI1_ROOT] = imx_clk_hw_gate4("ecspi1_root_clk", "ecspi1", ccm_base + 0x4070, 0); + hws[IMX8MP_CLK_ECSPI2_ROOT] = imx_clk_hw_gate4("ecspi2_root_clk", "ecspi2", ccm_base + 0x4080, 0); + hws[IMX8MP_CLK_ECSPI3_ROOT] = imx_clk_hw_gate4("ecspi3_root_clk", "ecspi3", ccm_base + 0x4090, 0); + hws[IMX8MP_CLK_ENET1_ROOT] = imx_clk_hw_gate4("enet1_root_clk", "enet_axi", ccm_base + 0x40a0, 0); + hws[IMX8MP_CLK_GPIO1_ROOT] = imx_clk_hw_gate4("gpio1_root_clk", "ipg_root", ccm_base + 0x40b0, 0); + hws[IMX8MP_CLK_GPIO2_ROOT] = imx_clk_hw_gate4("gpio2_root_clk", "ipg_root", ccm_base + 0x40c0, 0); + hws[IMX8MP_CLK_GPIO3_ROOT] = imx_clk_hw_gate4("gpio3_root_clk", "ipg_root", ccm_base + 0x40d0, 0); + hws[IMX8MP_CLK_GPIO4_ROOT] = imx_clk_hw_gate4("gpio4_root_clk", "ipg_root", ccm_base + 0x40e0, 0); + hws[IMX8MP_CLK_GPIO5_ROOT] = imx_clk_hw_gate4("gpio5_root_clk", "ipg_root", ccm_base + 0x40f0, 0); + hws[IMX8MP_CLK_GPT1_ROOT] = imx_clk_hw_gate4("gpt1_root_clk", "gpt1", ccm_base + 0x4100, 0); + hws[IMX8MP_CLK_GPT2_ROOT] = imx_clk_hw_gate4("gpt2_root_clk", "gpt2", ccm_base + 0x4110, 0); + hws[IMX8MP_CLK_GPT3_ROOT] = imx_clk_hw_gate4("gpt3_root_clk", "gpt3", ccm_base + 0x4120, 0); + hws[IMX8MP_CLK_GPT4_ROOT] = imx_clk_hw_gate4("gpt4_root_clk", "gpt4", ccm_base + 0x4130, 0); + hws[IMX8MP_CLK_GPT5_ROOT] = imx_clk_hw_gate4("gpt5_root_clk", "gpt5", ccm_base + 0x4140, 0); + hws[IMX8MP_CLK_GPT6_ROOT] = imx_clk_hw_gate4("gpt6_root_clk", "gpt6", ccm_base + 0x4150, 0); + hws[IMX8MP_CLK_I2C1_ROOT] = imx_clk_hw_gate4("i2c1_root_clk", "i2c1", ccm_base + 0x4170, 0); + hws[IMX8MP_CLK_I2C2_ROOT] = imx_clk_hw_gate4("i2c2_root_clk", "i2c2", ccm_base + 0x4180, 0); + hws[IMX8MP_CLK_I2C3_ROOT] = imx_clk_hw_gate4("i2c3_root_clk", "i2c3", ccm_base + 0x4190, 0); + hws[IMX8MP_CLK_I2C4_ROOT] = imx_clk_hw_gate4("i2c4_root_clk", "i2c4", ccm_base + 0x41a0, 0); + hws[IMX8MP_CLK_PCIE_ROOT] = imx_clk_hw_gate4("pcie_root_clk", "pcie_aux", ccm_base + 0x4250, 0); + hws[IMX8MP_CLK_PWM1_ROOT] = imx_clk_hw_gate4("pwm1_root_clk", "pwm1", ccm_base + 0x4280, 0); + hws[IMX8MP_CLK_PWM2_ROOT] = imx_clk_hw_gate4("pwm2_root_clk", "pwm2", ccm_base + 0x4290, 0); + hws[IMX8MP_CLK_PWM3_ROOT] = imx_clk_hw_gate4("pwm3_root_clk", "pwm3", ccm_base + 0x42a0, 0); + hws[IMX8MP_CLK_PWM4_ROOT] = imx_clk_hw_gate4("pwm4_root_clk", "pwm4", ccm_base + 0x42b0, 0); + hws[IMX8MP_CLK_QOS_ROOT] = imx_clk_hw_gate4("qos_root_clk", "ipg_root", ccm_base + 0x42c0, 0); + hws[IMX8MP_CLK_QOS_ENET_ROOT] = imx_clk_hw_gate4("qos_enet_root_clk", "ipg_root", ccm_base + 0x42e0, 0); + hws[IMX8MP_CLK_QSPI_ROOT] = imx_clk_hw_gate4("qspi_root_clk", "qspi", ccm_base + 0x42f0, 0); + hws[IMX8MP_CLK_NAND_ROOT] = imx_clk_hw_gate2_shared2("nand_root_clk", "nand", ccm_base + 0x4300, 0, &share_count_nand); + hws[IMX8MP_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", ccm_base + 0x4300, 0, &share_count_nand); + hws[IMX8MP_CLK_I2C5_ROOT] = imx_clk_hw_gate2("i2c5_root_clk", "i2c5", ccm_base + 0x4330, 0); + hws[IMX8MP_CLK_I2C6_ROOT] = imx_clk_hw_gate2("i2c6_root_clk", "i2c6", ccm_base + 0x4340, 0); + hws[IMX8MP_CLK_CAN1_ROOT] = imx_clk_hw_gate2("can1_root_clk", "can1", ccm_base + 0x4350, 0); + hws[IMX8MP_CLK_CAN2_ROOT] = imx_clk_hw_gate2("can2_root_clk", "can2", ccm_base + 0x4360, 0); + hws[IMX8MP_CLK_SDMA1_ROOT] = imx_clk_hw_gate4("sdma1_root_clk", "ipg_root", ccm_base + 0x43a0, 0); + hws[IMX8MP_CLK_ENET_QOS_ROOT] = imx_clk_hw_gate4("enet_qos_root_clk", "enet_axi", ccm_base + 0x43b0, 0); + hws[IMX8MP_CLK_SIM_ENET_ROOT] = imx_clk_hw_gate4("sim_enet_root_clk", "enet_axi", ccm_base + 0x4400, 0); + hws[IMX8MP_CLK_GPU2D_ROOT] = imx_clk_hw_gate4("gpu2d_root_clk", "gpu2d_div", ccm_base + 0x4450, 0); + hws[IMX8MP_CLK_GPU3D_ROOT] = imx_clk_hw_gate4("gpu3d_root_clk", "gpu3d_core_div", ccm_base + 0x4460, 0); + hws[IMX8MP_CLK_SNVS_ROOT] = imx_clk_hw_gate4("snvs_root_clk", "ipg_root", ccm_base + 0x4470, 0); + hws[IMX8MP_CLK_UART1_ROOT] = imx_clk_hw_gate4("uart1_root_clk", "uart1", ccm_base + 0x4490, 0); + hws[IMX8MP_CLK_UART2_ROOT] = imx_clk_hw_gate4("uart2_root_clk", "uart2", ccm_base + 0x44a0, 0); + hws[IMX8MP_CLK_UART3_ROOT] = imx_clk_hw_gate4("uart3_root_clk", "uart3", ccm_base + 0x44b0, 0); + hws[IMX8MP_CLK_UART4_ROOT] = imx_clk_hw_gate4("uart4_root_clk", "uart4", ccm_base + 0x44c0, 0); + hws[IMX8MP_CLK_USB_ROOT] = imx_clk_hw_gate4("usb_root_clk", "osc_32k", ccm_base + 0x44d0, 0); + hws[IMX8MP_CLK_USB_PHY_ROOT] = imx_clk_hw_gate4("usb_phy_root_clk", "usb_phy_ref", ccm_base + 0x44f0, 0); + hws[IMX8MP_CLK_USDHC1_ROOT] = imx_clk_hw_gate4("usdhc1_root_clk", "usdhc1", ccm_base + 0x4510, 0); + hws[IMX8MP_CLK_USDHC2_ROOT] = imx_clk_hw_gate4("usdhc2_root_clk", "usdhc2", ccm_base + 0x4520, 0); + hws[IMX8MP_CLK_WDOG1_ROOT] = imx_clk_hw_gate4("wdog1_root_clk", "wdog", ccm_base + 0x4530, 0); + hws[IMX8MP_CLK_WDOG2_ROOT] = imx_clk_hw_gate4("wdog2_root_clk", "wdog", ccm_base + 0x4540, 0); + hws[IMX8MP_CLK_WDOG3_ROOT] = imx_clk_hw_gate4("wdog3_root_clk", "wdog", ccm_base + 0x4550, 0); + hws[IMX8MP_CLK_VPU_G1_ROOT] = imx_clk_hw_gate4("vpu_g1_root_clk", "vpu_g1", ccm_base + 0x4560, 0); + hws[IMX8MP_CLK_GPU_ROOT] = imx_clk_hw_gate4("gpu_root_clk", "gpu_axi", ccm_base + 0x4570, 0); + hws[IMX8MP_CLK_VPU_VC8KE_ROOT] = imx_clk_hw_gate4("vpu_vc8ke_root_clk", "vpu_vc8000e", ccm_base + 0x4590, 0); + hws[IMX8MP_CLK_VPU_G2_ROOT] = imx_clk_hw_gate4("vpu_g2_root_clk", "vpu_g2", ccm_base + 0x45a0, 0); + hws[IMX8MP_CLK_NPU_ROOT] = imx_clk_hw_gate4("npu_root_clk", "ml_div", ccm_base + 0x45b0, 0); + hws[IMX8MP_CLK_HSIO_ROOT] = imx_clk_hw_gate4("hsio_root_clk", "ipg_root", ccm_base + 0x45c0, 0); + hws[IMX8MP_CLK_MEDIA_APB_ROOT] = imx_clk_hw_gate2_shared2("media_apb_root_clk", "media_apb", ccm_base + 0x45d0, 0, &share_count_media); + hws[IMX8MP_CLK_MEDIA_AXI_ROOT] = imx_clk_hw_gate2_shared2("media_axi_root_clk", "media_axi", ccm_base + 0x45d0, 0, &share_count_media); + hws[IMX8MP_CLK_MEDIA_CAM1_PIX_ROOT] = imx_clk_hw_gate2_shared2("media_cam1_pix_root_clk", "media_cam1_pix", ccm_base + 0x45d0, 0, &share_count_media); + hws[IMX8MP_CLK_MEDIA_CAM2_PIX_ROOT] = imx_clk_hw_gate2_shared2("media_cam2_pix_root_clk", "media_cam2_pix", ccm_base + 0x45d0, 0, &share_count_media); + hws[IMX8MP_CLK_MEDIA_DISP1_PIX_ROOT] = imx_clk_hw_gate2_shared2("media_disp1_pix_root_clk", "media_disp1_pix", ccm_base + 0x45d0, 0, &share_count_media); + hws[IMX8MP_CLK_MEDIA_DISP2_PIX_ROOT] = imx_clk_hw_gate2_shared2("media_disp2_pix_root_clk", "media_disp2_pix", ccm_base + 0x45d0, 0, &share_count_media); + hws[IMX8MP_CLK_MEDIA_ISP_ROOT] = imx_clk_hw_gate2_shared2("media_isp_root_clk", "media_isp_div", ccm_base + 0x45d0, 0, &share_count_media); + + hws[IMX8MP_CLK_USDHC3_ROOT] = imx_clk_hw_gate4("usdhc3_root_clk", "usdhc3", ccm_base + 0x45e0, 0); + hws[IMX8MP_CLK_HDMI_ROOT] = imx_clk_hw_gate4("hdmi_root_clk", "hdmi_axi", ccm_base + 0x45f0, 0); + hws[IMX8MP_CLK_TSENSOR_ROOT] = imx_clk_hw_gate4("tsensor_root_clk", "ipg_root", ccm_base + 0x4620, 0); + hws[IMX8MP_CLK_VPU_ROOT] = imx_clk_hw_gate4("vpu_root_clk", "vpu_bus", ccm_base + 0x4630, 0); + hws[IMX8MP_CLK_AUDIO_ROOT] = imx_clk_hw_gate4("audio_root_clk", "ipg_root", ccm_base + 0x4650, 0); + + hws[IMX8MP_CLK_ARM] = imx_clk_hw_cpu("arm", "arm_a53_div", + hws[IMX8MP_CLK_A53_DIV]->clk, + hws[IMX8MP_CLK_A53_SRC]->clk, + hws[IMX8MP_ARM_PLL_OUT]->clk, + hws[IMX8MP_SYS_PLL1_800M]->clk); + + imx_check_clk_hws(hws, IMX8MP_CLK_END); + + of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_clks[i] = &hws[index]->clk; + } + + imx_register_uart_clocks(uart_clks); + + return 0; +} + +static const struct of_device_id imx8mp_clk_of_match[] = { + { .compatible = "fsl,imx8mp-ccm" }, + { /* Sentinel */ } +}; +MODULE_DEVICE_TABLE(of, imx8mp_clk_of_match); + +static struct platform_driver imx8mp_clk_driver = { + .probe = imx8mp_clocks_probe, + .driver = { + .name = "imx8mp-ccm", + /* + * Disable bind attributes: clocks are not removed and + * reloading the driver will crash or break devices. + */ + .suppress_bind_attrs = true, + .of_match_table = of_match_ptr(imx8mp_clk_of_match), + }, +}; +module_platform_driver(imx8mp_clk_driver); diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index 5f10a606d836..4c0edca1a6d0 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -11,6 +11,7 @@ #include <linux/module.h> #include <linux/of_address.h> #include <linux/types.h> +#include <linux/slab.h> #include <linux/platform_device.h> #include "clk.h" @@ -24,8 +25,6 @@ static u32 share_count_sai6; static u32 share_count_dcss; static u32 share_count_nand; -static struct clk *clks[IMX8MQ_CLK_END]; - static const char * const pll_ref_sels[] = { "osc_25m", "osc_27m", "dummy", "dummy", }; static const char * const arm_pll_bypass_sels[] = {"arm_pll", "arm_pll_ref_sel", }; static const char * const gpu_pll_bypass_sels[] = {"gpu_pll", "gpu_pll_ref_sel", }; @@ -269,124 +268,133 @@ static const char * const imx8mq_clko1_sels[] = {"osc_25m", "sys1_pll_800m", "os static const char * const imx8mq_clko2_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_400m", "sys2_pll_166m", "sys3_pll_out", "audio_pll1_out", "video_pll1_out", "ckil", }; -static struct clk_onecell_data clk_data; +static struct clk_hw_onecell_data *clk_hw_data; +static struct clk_hw **hws; -static struct clk ** const uart_clks[] = { - &clks[IMX8MQ_CLK_UART1_ROOT], - &clks[IMX8MQ_CLK_UART2_ROOT], - &clks[IMX8MQ_CLK_UART3_ROOT], - &clks[IMX8MQ_CLK_UART4_ROOT], - NULL +static const int uart_clk_ids[] = { + IMX8MQ_CLK_UART1_ROOT, + IMX8MQ_CLK_UART2_ROOT, + IMX8MQ_CLK_UART3_ROOT, + IMX8MQ_CLK_UART4_ROOT, }; +static struct clk **uart_hws[ARRAY_SIZE(uart_clk_ids) + 1]; static int imx8mq_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; void __iomem *base; - int err; + int err, i; + + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX8MQ_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return -ENOMEM; + + clk_hw_data->num = IMX8MQ_CLK_END; + hws = clk_hw_data->hws; - clks[IMX8MQ_CLK_DUMMY] = imx_clk_fixed("dummy", 0); - clks[IMX8MQ_CLK_32K] = of_clk_get_by_name(np, "ckil"); - clks[IMX8MQ_CLK_25M] = of_clk_get_by_name(np, "osc_25m"); - clks[IMX8MQ_CLK_27M] = of_clk_get_by_name(np, "osc_27m"); - clks[IMX8MQ_CLK_EXT1] = of_clk_get_by_name(np, "clk_ext1"); - clks[IMX8MQ_CLK_EXT2] = of_clk_get_by_name(np, "clk_ext2"); - clks[IMX8MQ_CLK_EXT3] = of_clk_get_by_name(np, "clk_ext3"); - clks[IMX8MQ_CLK_EXT4] = of_clk_get_by_name(np, "clk_ext4"); + hws[IMX8MQ_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); + hws[IMX8MQ_CLK_32K] = imx_obtain_fixed_clk_hw(np, "ckil"); + hws[IMX8MQ_CLK_25M] = imx_obtain_fixed_clk_hw(np, "osc_25m"); + hws[IMX8MQ_CLK_27M] = imx_obtain_fixed_clk_hw(np, "osc_27m"); + hws[IMX8MQ_CLK_EXT1] = imx_obtain_fixed_clk_hw(np, "clk_ext1"); + hws[IMX8MQ_CLK_EXT2] = imx_obtain_fixed_clk_hw(np, "clk_ext2"); + hws[IMX8MQ_CLK_EXT3] = imx_obtain_fixed_clk_hw(np, "clk_ext3"); + hws[IMX8MQ_CLK_EXT4] = imx_obtain_fixed_clk_hw(np, "clk_ext4"); np = of_find_compatible_node(NULL, NULL, "fsl,imx8mq-anatop"); base = of_iomap(np, 0); if (WARN_ON(!base)) return -ENOMEM; - clks[IMX8MQ_ARM_PLL_REF_SEL] = imx_clk_mux("arm_pll_ref_sel", base + 0x28, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_GPU_PLL_REF_SEL] = imx_clk_mux("gpu_pll_ref_sel", base + 0x18, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_VPU_PLL_REF_SEL] = imx_clk_mux("vpu_pll_ref_sel", base + 0x20, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_AUDIO_PLL1_REF_SEL] = imx_clk_mux("audio_pll1_ref_sel", base + 0x0, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_AUDIO_PLL2_REF_SEL] = imx_clk_mux("audio_pll2_ref_sel", base + 0x8, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_VIDEO_PLL1_REF_SEL] = imx_clk_mux("video_pll1_ref_sel", base + 0x10, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_SYS3_PLL1_REF_SEL] = imx_clk_mux("sys3_pll1_ref_sel", base + 0x48, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_DRAM_PLL1_REF_SEL] = imx_clk_mux("dram_pll1_ref_sel", base + 0x60, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - clks[IMX8MQ_VIDEO2_PLL1_REF_SEL] = imx_clk_mux("video2_pll1_ref_sel", base + 0x54, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); - - clks[IMX8MQ_ARM_PLL_REF_DIV] = imx_clk_divider("arm_pll_ref_div", "arm_pll_ref_sel", base + 0x28, 5, 6); - clks[IMX8MQ_GPU_PLL_REF_DIV] = imx_clk_divider("gpu_pll_ref_div", "gpu_pll_ref_sel", base + 0x18, 5, 6); - clks[IMX8MQ_VPU_PLL_REF_DIV] = imx_clk_divider("vpu_pll_ref_div", "vpu_pll_ref_sel", base + 0x20, 5, 6); - clks[IMX8MQ_AUDIO_PLL1_REF_DIV] = imx_clk_divider("audio_pll1_ref_div", "audio_pll1_ref_sel", base + 0x0, 5, 6); - clks[IMX8MQ_AUDIO_PLL2_REF_DIV] = imx_clk_divider("audio_pll2_ref_div", "audio_pll2_ref_sel", base + 0x8, 5, 6); - clks[IMX8MQ_VIDEO_PLL1_REF_DIV] = imx_clk_divider("video_pll1_ref_div", "video_pll1_ref_sel", base + 0x10, 5, 6); - - clks[IMX8MQ_ARM_PLL] = imx_clk_frac_pll("arm_pll", "arm_pll_ref_div", base + 0x28); - clks[IMX8MQ_GPU_PLL] = imx_clk_frac_pll("gpu_pll", "gpu_pll_ref_div", base + 0x18); - clks[IMX8MQ_VPU_PLL] = imx_clk_frac_pll("vpu_pll", "vpu_pll_ref_div", base + 0x20); - clks[IMX8MQ_AUDIO_PLL1] = imx_clk_frac_pll("audio_pll1", "audio_pll1_ref_div", base + 0x0); - clks[IMX8MQ_AUDIO_PLL2] = imx_clk_frac_pll("audio_pll2", "audio_pll2_ref_div", base + 0x8); - clks[IMX8MQ_VIDEO_PLL1] = imx_clk_frac_pll("video_pll1", "video_pll1_ref_div", base + 0x10); + hws[IMX8MQ_ARM_PLL_REF_SEL] = imx_clk_hw_mux("arm_pll_ref_sel", base + 0x28, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_GPU_PLL_REF_SEL] = imx_clk_hw_mux("gpu_pll_ref_sel", base + 0x18, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_VPU_PLL_REF_SEL] = imx_clk_hw_mux("vpu_pll_ref_sel", base + 0x20, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_AUDIO_PLL1_REF_SEL] = imx_clk_hw_mux("audio_pll1_ref_sel", base + 0x0, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_AUDIO_PLL2_REF_SEL] = imx_clk_hw_mux("audio_pll2_ref_sel", base + 0x8, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_VIDEO_PLL1_REF_SEL] = imx_clk_hw_mux("video_pll1_ref_sel", base + 0x10, 16, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_SYS3_PLL1_REF_SEL] = imx_clk_hw_mux("sys3_pll1_ref_sel", base + 0x48, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_DRAM_PLL1_REF_SEL] = imx_clk_hw_mux("dram_pll1_ref_sel", base + 0x60, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + hws[IMX8MQ_VIDEO2_PLL1_REF_SEL] = imx_clk_hw_mux("video2_pll1_ref_sel", base + 0x54, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels)); + + hws[IMX8MQ_ARM_PLL_REF_DIV] = imx_clk_hw_divider("arm_pll_ref_div", "arm_pll_ref_sel", base + 0x28, 5, 6); + hws[IMX8MQ_GPU_PLL_REF_DIV] = imx_clk_hw_divider("gpu_pll_ref_div", "gpu_pll_ref_sel", base + 0x18, 5, 6); + hws[IMX8MQ_VPU_PLL_REF_DIV] = imx_clk_hw_divider("vpu_pll_ref_div", "vpu_pll_ref_sel", base + 0x20, 5, 6); + hws[IMX8MQ_AUDIO_PLL1_REF_DIV] = imx_clk_hw_divider("audio_pll1_ref_div", "audio_pll1_ref_sel", base + 0x0, 5, 6); + hws[IMX8MQ_AUDIO_PLL2_REF_DIV] = imx_clk_hw_divider("audio_pll2_ref_div", "audio_pll2_ref_sel", base + 0x8, 5, 6); + hws[IMX8MQ_VIDEO_PLL1_REF_DIV] = imx_clk_hw_divider("video_pll1_ref_div", "video_pll1_ref_sel", base + 0x10, 5, 6); + + hws[IMX8MQ_ARM_PLL] = imx_clk_hw_frac_pll("arm_pll", "arm_pll_ref_div", base + 0x28); + hws[IMX8MQ_GPU_PLL] = imx_clk_hw_frac_pll("gpu_pll", "gpu_pll_ref_div", base + 0x18); + hws[IMX8MQ_VPU_PLL] = imx_clk_hw_frac_pll("vpu_pll", "vpu_pll_ref_div", base + 0x20); + hws[IMX8MQ_AUDIO_PLL1] = imx_clk_hw_frac_pll("audio_pll1", "audio_pll1_ref_div", base + 0x0); + hws[IMX8MQ_AUDIO_PLL2] = imx_clk_hw_frac_pll("audio_pll2", "audio_pll2_ref_div", base + 0x8); + hws[IMX8MQ_VIDEO_PLL1] = imx_clk_hw_frac_pll("video_pll1", "video_pll1_ref_div", base + 0x10); /* PLL bypass out */ - clks[IMX8MQ_ARM_PLL_BYPASS] = imx_clk_mux_flags("arm_pll_bypass", base + 0x28, 14, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX8MQ_GPU_PLL_BYPASS] = imx_clk_mux("gpu_pll_bypass", base + 0x18, 14, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels)); - clks[IMX8MQ_VPU_PLL_BYPASS] = imx_clk_mux("vpu_pll_bypass", base + 0x20, 14, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels)); - clks[IMX8MQ_AUDIO_PLL1_BYPASS] = imx_clk_mux("audio_pll1_bypass", base + 0x0, 14, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels)); - clks[IMX8MQ_AUDIO_PLL2_BYPASS] = imx_clk_mux("audio_pll2_bypass", base + 0x8, 14, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels)); - clks[IMX8MQ_VIDEO_PLL1_BYPASS] = imx_clk_mux("video_pll1_bypass", base + 0x10, 14, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels)); + hws[IMX8MQ_ARM_PLL_BYPASS] = imx_clk_hw_mux_flags("arm_pll_bypass", base + 0x28, 14, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX8MQ_GPU_PLL_BYPASS] = imx_clk_hw_mux("gpu_pll_bypass", base + 0x18, 14, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels)); + hws[IMX8MQ_VPU_PLL_BYPASS] = imx_clk_hw_mux("vpu_pll_bypass", base + 0x20, 14, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels)); + hws[IMX8MQ_AUDIO_PLL1_BYPASS] = imx_clk_hw_mux("audio_pll1_bypass", base + 0x0, 14, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels)); + hws[IMX8MQ_AUDIO_PLL2_BYPASS] = imx_clk_hw_mux("audio_pll2_bypass", base + 0x8, 14, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels)); + hws[IMX8MQ_VIDEO_PLL1_BYPASS] = imx_clk_hw_mux("video_pll1_bypass", base + 0x10, 14, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels)); /* PLL OUT GATE */ - clks[IMX8MQ_ARM_PLL_OUT] = imx_clk_gate("arm_pll_out", "arm_pll_bypass", base + 0x28, 21); - clks[IMX8MQ_GPU_PLL_OUT] = imx_clk_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x18, 21); - clks[IMX8MQ_VPU_PLL_OUT] = imx_clk_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x20, 21); - clks[IMX8MQ_AUDIO_PLL1_OUT] = imx_clk_gate("audio_pll1_out", "audio_pll1_bypass", base + 0x0, 21); - clks[IMX8MQ_AUDIO_PLL2_OUT] = imx_clk_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x8, 21); - clks[IMX8MQ_VIDEO_PLL1_OUT] = imx_clk_gate("video_pll1_out", "video_pll1_bypass", base + 0x10, 21); - - clks[IMX8MQ_SYS1_PLL_OUT] = imx_clk_fixed("sys1_pll_out", 800000000); - clks[IMX8MQ_SYS2_PLL_OUT] = imx_clk_fixed("sys2_pll_out", 1000000000); - clks[IMX8MQ_SYS3_PLL_OUT] = imx_clk_sccg_pll("sys3_pll_out", sys3_pll_out_sels, ARRAY_SIZE(sys3_pll_out_sels), 0, 0, 0, base + 0x48, CLK_IS_CRITICAL); - clks[IMX8MQ_DRAM_PLL_OUT] = imx_clk_sccg_pll("dram_pll_out", dram_pll_out_sels, ARRAY_SIZE(dram_pll_out_sels), 0, 0, 0, base + 0x60, CLK_IS_CRITICAL); - clks[IMX8MQ_VIDEO2_PLL_OUT] = imx_clk_sccg_pll("video2_pll_out", video2_pll_out_sels, ARRAY_SIZE(video2_pll_out_sels), 0, 0, 0, base + 0x54, 0); + hws[IMX8MQ_ARM_PLL_OUT] = imx_clk_hw_gate("arm_pll_out", "arm_pll_bypass", base + 0x28, 21); + hws[IMX8MQ_GPU_PLL_OUT] = imx_clk_hw_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x18, 21); + hws[IMX8MQ_VPU_PLL_OUT] = imx_clk_hw_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x20, 21); + hws[IMX8MQ_AUDIO_PLL1_OUT] = imx_clk_hw_gate("audio_pll1_out", "audio_pll1_bypass", base + 0x0, 21); + hws[IMX8MQ_AUDIO_PLL2_OUT] = imx_clk_hw_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x8, 21); + hws[IMX8MQ_VIDEO_PLL1_OUT] = imx_clk_hw_gate("video_pll1_out", "video_pll1_bypass", base + 0x10, 21); + + hws[IMX8MQ_SYS1_PLL_OUT] = imx_clk_hw_fixed("sys1_pll_out", 800000000); + hws[IMX8MQ_SYS2_PLL_OUT] = imx_clk_hw_fixed("sys2_pll_out", 1000000000); + hws[IMX8MQ_SYS3_PLL_OUT] = imx_clk_hw_sscg_pll("sys3_pll_out", sys3_pll_out_sels, ARRAY_SIZE(sys3_pll_out_sels), 0, 0, 0, base + 0x48, CLK_IS_CRITICAL); + hws[IMX8MQ_DRAM_PLL_OUT] = imx_clk_hw_sscg_pll("dram_pll_out", dram_pll_out_sels, ARRAY_SIZE(dram_pll_out_sels), 0, 0, 0, base + 0x60, CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); + hws[IMX8MQ_VIDEO2_PLL_OUT] = imx_clk_hw_sscg_pll("video2_pll_out", video2_pll_out_sels, ARRAY_SIZE(video2_pll_out_sels), 0, 0, 0, base + 0x54, 0); /* SYS PLL1 fixed output */ - clks[IMX8MQ_SYS1_PLL_40M_CG] = imx_clk_gate("sys1_pll_40m_cg", "sys1_pll_out", base + 0x30, 9); - clks[IMX8MQ_SYS1_PLL_80M_CG] = imx_clk_gate("sys1_pll_80m_cg", "sys1_pll_out", base + 0x30, 11); - clks[IMX8MQ_SYS1_PLL_100M_CG] = imx_clk_gate("sys1_pll_100m_cg", "sys1_pll_out", base + 0x30, 13); - clks[IMX8MQ_SYS1_PLL_133M_CG] = imx_clk_gate("sys1_pll_133m_cg", "sys1_pll_out", base + 0x30, 15); - clks[IMX8MQ_SYS1_PLL_160M_CG] = imx_clk_gate("sys1_pll_160m_cg", "sys1_pll_out", base + 0x30, 17); - clks[IMX8MQ_SYS1_PLL_200M_CG] = imx_clk_gate("sys1_pll_200m_cg", "sys1_pll_out", base + 0x30, 19); - clks[IMX8MQ_SYS1_PLL_266M_CG] = imx_clk_gate("sys1_pll_266m_cg", "sys1_pll_out", base + 0x30, 21); - clks[IMX8MQ_SYS1_PLL_400M_CG] = imx_clk_gate("sys1_pll_400m_cg", "sys1_pll_out", base + 0x30, 23); - clks[IMX8MQ_SYS1_PLL_800M_CG] = imx_clk_gate("sys1_pll_800m_cg", "sys1_pll_out", base + 0x30, 25); - - clks[IMX8MQ_SYS1_PLL_40M] = imx_clk_fixed_factor("sys1_pll_40m", "sys1_pll_40m_cg", 1, 20); - clks[IMX8MQ_SYS1_PLL_80M] = imx_clk_fixed_factor("sys1_pll_80m", "sys1_pll_80m_cg", 1, 10); - clks[IMX8MQ_SYS1_PLL_100M] = imx_clk_fixed_factor("sys1_pll_100m", "sys1_pll_100m_cg", 1, 8); - clks[IMX8MQ_SYS1_PLL_133M] = imx_clk_fixed_factor("sys1_pll_133m", "sys1_pll_133m_cg", 1, 6); - clks[IMX8MQ_SYS1_PLL_160M] = imx_clk_fixed_factor("sys1_pll_160m", "sys1_pll_160m_cg", 1, 5); - clks[IMX8MQ_SYS1_PLL_200M] = imx_clk_fixed_factor("sys1_pll_200m", "sys1_pll_200m_cg", 1, 4); - clks[IMX8MQ_SYS1_PLL_266M] = imx_clk_fixed_factor("sys1_pll_266m", "sys1_pll_266m_cg", 1, 3); - clks[IMX8MQ_SYS1_PLL_400M] = imx_clk_fixed_factor("sys1_pll_400m", "sys1_pll_400m_cg", 1, 2); - clks[IMX8MQ_SYS1_PLL_800M] = imx_clk_fixed_factor("sys1_pll_800m", "sys1_pll_800m_cg", 1, 1); + hws[IMX8MQ_SYS1_PLL_40M_CG] = imx_clk_hw_gate("sys1_pll_40m_cg", "sys1_pll_out", base + 0x30, 9); + hws[IMX8MQ_SYS1_PLL_80M_CG] = imx_clk_hw_gate("sys1_pll_80m_cg", "sys1_pll_out", base + 0x30, 11); + hws[IMX8MQ_SYS1_PLL_100M_CG] = imx_clk_hw_gate("sys1_pll_100m_cg", "sys1_pll_out", base + 0x30, 13); + hws[IMX8MQ_SYS1_PLL_133M_CG] = imx_clk_hw_gate("sys1_pll_133m_cg", "sys1_pll_out", base + 0x30, 15); + hws[IMX8MQ_SYS1_PLL_160M_CG] = imx_clk_hw_gate("sys1_pll_160m_cg", "sys1_pll_out", base + 0x30, 17); + hws[IMX8MQ_SYS1_PLL_200M_CG] = imx_clk_hw_gate("sys1_pll_200m_cg", "sys1_pll_out", base + 0x30, 19); + hws[IMX8MQ_SYS1_PLL_266M_CG] = imx_clk_hw_gate("sys1_pll_266m_cg", "sys1_pll_out", base + 0x30, 21); + hws[IMX8MQ_SYS1_PLL_400M_CG] = imx_clk_hw_gate("sys1_pll_400m_cg", "sys1_pll_out", base + 0x30, 23); + hws[IMX8MQ_SYS1_PLL_800M_CG] = imx_clk_hw_gate("sys1_pll_800m_cg", "sys1_pll_out", base + 0x30, 25); + + hws[IMX8MQ_SYS1_PLL_40M] = imx_clk_hw_fixed_factor("sys1_pll_40m", "sys1_pll_40m_cg", 1, 20); + hws[IMX8MQ_SYS1_PLL_80M] = imx_clk_hw_fixed_factor("sys1_pll_80m", "sys1_pll_80m_cg", 1, 10); + hws[IMX8MQ_SYS1_PLL_100M] = imx_clk_hw_fixed_factor("sys1_pll_100m", "sys1_pll_100m_cg", 1, 8); + hws[IMX8MQ_SYS1_PLL_133M] = imx_clk_hw_fixed_factor("sys1_pll_133m", "sys1_pll_133m_cg", 1, 6); + hws[IMX8MQ_SYS1_PLL_160M] = imx_clk_hw_fixed_factor("sys1_pll_160m", "sys1_pll_160m_cg", 1, 5); + hws[IMX8MQ_SYS1_PLL_200M] = imx_clk_hw_fixed_factor("sys1_pll_200m", "sys1_pll_200m_cg", 1, 4); + hws[IMX8MQ_SYS1_PLL_266M] = imx_clk_hw_fixed_factor("sys1_pll_266m", "sys1_pll_266m_cg", 1, 3); + hws[IMX8MQ_SYS1_PLL_400M] = imx_clk_hw_fixed_factor("sys1_pll_400m", "sys1_pll_400m_cg", 1, 2); + hws[IMX8MQ_SYS1_PLL_800M] = imx_clk_hw_fixed_factor("sys1_pll_800m", "sys1_pll_800m_cg", 1, 1); /* SYS PLL2 fixed output */ - clks[IMX8MQ_SYS2_PLL_50M_CG] = imx_clk_gate("sys2_pll_50m_cg", "sys2_pll_out", base + 0x3c, 9); - clks[IMX8MQ_SYS2_PLL_100M_CG] = imx_clk_gate("sys2_pll_100m_cg", "sys2_pll_out", base + 0x3c, 11); - clks[IMX8MQ_SYS2_PLL_125M_CG] = imx_clk_gate("sys2_pll_125m_cg", "sys2_pll_out", base + 0x3c, 13); - clks[IMX8MQ_SYS2_PLL_166M_CG] = imx_clk_gate("sys2_pll_166m_cg", "sys2_pll_out", base + 0x3c, 15); - clks[IMX8MQ_SYS2_PLL_200M_CG] = imx_clk_gate("sys2_pll_200m_cg", "sys2_pll_out", base + 0x3c, 17); - clks[IMX8MQ_SYS2_PLL_250M_CG] = imx_clk_gate("sys2_pll_250m_cg", "sys2_pll_out", base + 0x3c, 19); - clks[IMX8MQ_SYS2_PLL_333M_CG] = imx_clk_gate("sys2_pll_333m_cg", "sys2_pll_out", base + 0x3c, 21); - clks[IMX8MQ_SYS2_PLL_500M_CG] = imx_clk_gate("sys2_pll_500m_cg", "sys2_pll_out", base + 0x3c, 23); - clks[IMX8MQ_SYS2_PLL_1000M_CG] = imx_clk_gate("sys2_pll_1000m_cg", "sys2_pll_out", base + 0x3c, 25); - - clks[IMX8MQ_SYS2_PLL_50M] = imx_clk_fixed_factor("sys2_pll_50m", "sys2_pll_50m_cg", 1, 20); - clks[IMX8MQ_SYS2_PLL_100M] = imx_clk_fixed_factor("sys2_pll_100m", "sys2_pll_100m_cg", 1, 10); - clks[IMX8MQ_SYS2_PLL_125M] = imx_clk_fixed_factor("sys2_pll_125m", "sys2_pll_125m_cg", 1, 8); - clks[IMX8MQ_SYS2_PLL_166M] = imx_clk_fixed_factor("sys2_pll_166m", "sys2_pll_166m_cg", 1, 6); - clks[IMX8MQ_SYS2_PLL_200M] = imx_clk_fixed_factor("sys2_pll_200m", "sys2_pll_200m_cg", 1, 5); - clks[IMX8MQ_SYS2_PLL_250M] = imx_clk_fixed_factor("sys2_pll_250m", "sys2_pll_250m_cg", 1, 4); - clks[IMX8MQ_SYS2_PLL_333M] = imx_clk_fixed_factor("sys2_pll_333m", "sys2_pll_333m_cg", 1, 3); - clks[IMX8MQ_SYS2_PLL_500M] = imx_clk_fixed_factor("sys2_pll_500m", "sys2_pll_500m_cg", 1, 2); - clks[IMX8MQ_SYS2_PLL_1000M] = imx_clk_fixed_factor("sys2_pll_1000m", "sys2_pll_1000m_cg", 1, 1); + hws[IMX8MQ_SYS2_PLL_50M_CG] = imx_clk_hw_gate("sys2_pll_50m_cg", "sys2_pll_out", base + 0x3c, 9); + hws[IMX8MQ_SYS2_PLL_100M_CG] = imx_clk_hw_gate("sys2_pll_100m_cg", "sys2_pll_out", base + 0x3c, 11); + hws[IMX8MQ_SYS2_PLL_125M_CG] = imx_clk_hw_gate("sys2_pll_125m_cg", "sys2_pll_out", base + 0x3c, 13); + hws[IMX8MQ_SYS2_PLL_166M_CG] = imx_clk_hw_gate("sys2_pll_166m_cg", "sys2_pll_out", base + 0x3c, 15); + hws[IMX8MQ_SYS2_PLL_200M_CG] = imx_clk_hw_gate("sys2_pll_200m_cg", "sys2_pll_out", base + 0x3c, 17); + hws[IMX8MQ_SYS2_PLL_250M_CG] = imx_clk_hw_gate("sys2_pll_250m_cg", "sys2_pll_out", base + 0x3c, 19); + hws[IMX8MQ_SYS2_PLL_333M_CG] = imx_clk_hw_gate("sys2_pll_333m_cg", "sys2_pll_out", base + 0x3c, 21); + hws[IMX8MQ_SYS2_PLL_500M_CG] = imx_clk_hw_gate("sys2_pll_500m_cg", "sys2_pll_out", base + 0x3c, 23); + hws[IMX8MQ_SYS2_PLL_1000M_CG] = imx_clk_hw_gate("sys2_pll_1000m_cg", "sys2_pll_out", base + 0x3c, 25); + + hws[IMX8MQ_SYS2_PLL_50M] = imx_clk_hw_fixed_factor("sys2_pll_50m", "sys2_pll_50m_cg", 1, 20); + hws[IMX8MQ_SYS2_PLL_100M] = imx_clk_hw_fixed_factor("sys2_pll_100m", "sys2_pll_100m_cg", 1, 10); + hws[IMX8MQ_SYS2_PLL_125M] = imx_clk_hw_fixed_factor("sys2_pll_125m", "sys2_pll_125m_cg", 1, 8); + hws[IMX8MQ_SYS2_PLL_166M] = imx_clk_hw_fixed_factor("sys2_pll_166m", "sys2_pll_166m_cg", 1, 6); + hws[IMX8MQ_SYS2_PLL_200M] = imx_clk_hw_fixed_factor("sys2_pll_200m", "sys2_pll_200m_cg", 1, 5); + hws[IMX8MQ_SYS2_PLL_250M] = imx_clk_hw_fixed_factor("sys2_pll_250m", "sys2_pll_250m_cg", 1, 4); + hws[IMX8MQ_SYS2_PLL_333M] = imx_clk_hw_fixed_factor("sys2_pll_333m", "sys2_pll_333m_cg", 1, 3); + hws[IMX8MQ_SYS2_PLL_500M] = imx_clk_hw_fixed_factor("sys2_pll_500m", "sys2_pll_500m_cg", 1, 2); + hws[IMX8MQ_SYS2_PLL_1000M] = imx_clk_hw_fixed_factor("sys2_pll_1000m", "sys2_pll_1000m_cg", 1, 1); np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); @@ -394,206 +402,213 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) return PTR_ERR(base); /* CORE */ - clks[IMX8MQ_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mq_a53_sels, ARRAY_SIZE(imx8mq_a53_sels)); - clks[IMX8MQ_CLK_M4_SRC] = imx_clk_mux2("arm_m4_src", base + 0x8080, 24, 3, imx8mq_arm_m4_sels, ARRAY_SIZE(imx8mq_arm_m4_sels)); - clks[IMX8MQ_CLK_VPU_SRC] = imx_clk_mux2("vpu_src", base + 0x8100, 24, 3, imx8mq_vpu_sels, ARRAY_SIZE(imx8mq_vpu_sels)); - clks[IMX8MQ_CLK_GPU_CORE_SRC] = imx_clk_mux2("gpu_core_src", base + 0x8180, 24, 3, imx8mq_gpu_core_sels, ARRAY_SIZE(imx8mq_gpu_core_sels)); - clks[IMX8MQ_CLK_GPU_SHADER_SRC] = imx_clk_mux2("gpu_shader_src", base + 0x8200, 24, 3, imx8mq_gpu_shader_sels, ARRAY_SIZE(imx8mq_gpu_shader_sels)); - - clks[IMX8MQ_CLK_A53_CG] = imx_clk_gate3_flags("arm_a53_cg", "arm_a53_src", base + 0x8000, 28, CLK_IS_CRITICAL); - clks[IMX8MQ_CLK_M4_CG] = imx_clk_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); - clks[IMX8MQ_CLK_VPU_CG] = imx_clk_gate3("vpu_cg", "vpu_src", base + 0x8100, 28); - clks[IMX8MQ_CLK_GPU_CORE_CG] = imx_clk_gate3("gpu_core_cg", "gpu_core_src", base + 0x8180, 28); - clks[IMX8MQ_CLK_GPU_SHADER_CG] = imx_clk_gate3("gpu_shader_cg", "gpu_shader_src", base + 0x8200, 28); - - clks[IMX8MQ_CLK_A53_DIV] = imx_clk_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); - clks[IMX8MQ_CLK_M4_DIV] = imx_clk_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); - clks[IMX8MQ_CLK_VPU_DIV] = imx_clk_divider2("vpu_div", "vpu_cg", base + 0x8100, 0, 3); - clks[IMX8MQ_CLK_GPU_CORE_DIV] = imx_clk_divider2("gpu_core_div", "gpu_core_cg", base + 0x8180, 0, 3); - clks[IMX8MQ_CLK_GPU_SHADER_DIV] = imx_clk_divider2("gpu_shader_div", "gpu_shader_cg", base + 0x8200, 0, 3); + hws[IMX8MQ_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mq_a53_sels, ARRAY_SIZE(imx8mq_a53_sels)); + hws[IMX8MQ_CLK_M4_SRC] = imx_clk_hw_mux2("arm_m4_src", base + 0x8080, 24, 3, imx8mq_arm_m4_sels, ARRAY_SIZE(imx8mq_arm_m4_sels)); + hws[IMX8MQ_CLK_VPU_SRC] = imx_clk_hw_mux2("vpu_src", base + 0x8100, 24, 3, imx8mq_vpu_sels, ARRAY_SIZE(imx8mq_vpu_sels)); + hws[IMX8MQ_CLK_GPU_CORE_SRC] = imx_clk_hw_mux2("gpu_core_src", base + 0x8180, 24, 3, imx8mq_gpu_core_sels, ARRAY_SIZE(imx8mq_gpu_core_sels)); + hws[IMX8MQ_CLK_GPU_SHADER_SRC] = imx_clk_hw_mux2("gpu_shader_src", base + 0x8200, 24, 3, imx8mq_gpu_shader_sels, ARRAY_SIZE(imx8mq_gpu_shader_sels)); + + hws[IMX8MQ_CLK_A53_CG] = imx_clk_hw_gate3_flags("arm_a53_cg", "arm_a53_src", base + 0x8000, 28, CLK_IS_CRITICAL); + hws[IMX8MQ_CLK_M4_CG] = imx_clk_hw_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); + hws[IMX8MQ_CLK_VPU_CG] = imx_clk_hw_gate3("vpu_cg", "vpu_src", base + 0x8100, 28); + hws[IMX8MQ_CLK_GPU_CORE_CG] = imx_clk_hw_gate3("gpu_core_cg", "gpu_core_src", base + 0x8180, 28); + hws[IMX8MQ_CLK_GPU_SHADER_CG] = imx_clk_hw_gate3("gpu_shader_cg", "gpu_shader_src", base + 0x8200, 28); + + hws[IMX8MQ_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3); + hws[IMX8MQ_CLK_M4_DIV] = imx_clk_hw_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); + hws[IMX8MQ_CLK_VPU_DIV] = imx_clk_hw_divider2("vpu_div", "vpu_cg", base + 0x8100, 0, 3); + hws[IMX8MQ_CLK_GPU_CORE_DIV] = imx_clk_hw_divider2("gpu_core_div", "gpu_core_cg", base + 0x8180, 0, 3); + hws[IMX8MQ_CLK_GPU_SHADER_DIV] = imx_clk_hw_divider2("gpu_shader_div", "gpu_shader_cg", base + 0x8200, 0, 3); /* BUS */ - clks[IMX8MQ_CLK_MAIN_AXI] = imx8m_clk_composite_critical("main_axi", imx8mq_main_axi_sels, base + 0x8800); - clks[IMX8MQ_CLK_ENET_AXI] = imx8m_clk_composite("enet_axi", imx8mq_enet_axi_sels, base + 0x8880); - clks[IMX8MQ_CLK_NAND_USDHC_BUS] = imx8m_clk_composite("nand_usdhc_bus", imx8mq_nand_usdhc_sels, base + 0x8900); - clks[IMX8MQ_CLK_VPU_BUS] = imx8m_clk_composite("vpu_bus", imx8mq_vpu_bus_sels, base + 0x8980); - clks[IMX8MQ_CLK_DISP_AXI] = imx8m_clk_composite("disp_axi", imx8mq_disp_axi_sels, base + 0x8a00); - clks[IMX8MQ_CLK_DISP_APB] = imx8m_clk_composite("disp_apb", imx8mq_disp_apb_sels, base + 0x8a80); - clks[IMX8MQ_CLK_DISP_RTRM] = imx8m_clk_composite("disp_rtrm", imx8mq_disp_rtrm_sels, base + 0x8b00); - clks[IMX8MQ_CLK_USB_BUS] = imx8m_clk_composite("usb_bus", imx8mq_usb_bus_sels, base + 0x8b80); - clks[IMX8MQ_CLK_GPU_AXI] = imx8m_clk_composite("gpu_axi", imx8mq_gpu_axi_sels, base + 0x8c00); - clks[IMX8MQ_CLK_GPU_AHB] = imx8m_clk_composite("gpu_ahb", imx8mq_gpu_ahb_sels, base + 0x8c80); - clks[IMX8MQ_CLK_NOC] = imx8m_clk_composite_critical("noc", imx8mq_noc_sels, base + 0x8d00); - clks[IMX8MQ_CLK_NOC_APB] = imx8m_clk_composite_critical("noc_apb", imx8mq_noc_apb_sels, base + 0x8d80); + hws[IMX8MQ_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi", imx8mq_main_axi_sels, base + 0x8800); + hws[IMX8MQ_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mq_enet_axi_sels, base + 0x8880); + hws[IMX8MQ_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite("nand_usdhc_bus", imx8mq_nand_usdhc_sels, base + 0x8900); + hws[IMX8MQ_CLK_VPU_BUS] = imx8m_clk_hw_composite("vpu_bus", imx8mq_vpu_bus_sels, base + 0x8980); + hws[IMX8MQ_CLK_DISP_AXI] = imx8m_clk_hw_composite("disp_axi", imx8mq_disp_axi_sels, base + 0x8a00); + hws[IMX8MQ_CLK_DISP_APB] = imx8m_clk_hw_composite("disp_apb", imx8mq_disp_apb_sels, base + 0x8a80); + hws[IMX8MQ_CLK_DISP_RTRM] = imx8m_clk_hw_composite("disp_rtrm", imx8mq_disp_rtrm_sels, base + 0x8b00); + hws[IMX8MQ_CLK_USB_BUS] = imx8m_clk_hw_composite("usb_bus", imx8mq_usb_bus_sels, base + 0x8b80); + hws[IMX8MQ_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mq_gpu_axi_sels, base + 0x8c00); + hws[IMX8MQ_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mq_gpu_ahb_sels, base + 0x8c80); + hws[IMX8MQ_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mq_noc_sels, base + 0x8d00); + hws[IMX8MQ_CLK_NOC_APB] = imx8m_clk_hw_composite_critical("noc_apb", imx8mq_noc_apb_sels, base + 0x8d80); /* AHB */ /* AHB clock is used by the AHB bus therefore marked as critical */ - clks[IMX8MQ_CLK_AHB] = imx8m_clk_composite_critical("ahb", imx8mq_ahb_sels, base + 0x9000); - clks[IMX8MQ_CLK_AUDIO_AHB] = imx8m_clk_composite("audio_ahb", imx8mq_audio_ahb_sels, base + 0x9100); + hws[IMX8MQ_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb", imx8mq_ahb_sels, base + 0x9000); + hws[IMX8MQ_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mq_audio_ahb_sels, base + 0x9100); /* IPG */ - clks[IMX8MQ_CLK_IPG_ROOT] = imx_clk_divider2("ipg_root", "ahb", base + 0x9080, 0, 1); - clks[IMX8MQ_CLK_IPG_AUDIO_ROOT] = imx_clk_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1); + hws[IMX8MQ_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb", base + 0x9080, 0, 1); + hws[IMX8MQ_CLK_IPG_AUDIO_ROOT] = imx_clk_hw_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1); + + /* + * DRAM clocks are manipulated from TF-A outside clock framework. + * Mark with GET_RATE_NOCACHE to always read div value from hardware + */ + hws[IMX8MQ_CLK_DRAM_CORE] = imx_clk_hw_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mq_dram_core_sels, ARRAY_SIZE(imx8mq_dram_core_sels), CLK_IS_CRITICAL); + hws[IMX8MQ_CLK_DRAM_ALT] = __imx8m_clk_hw_composite("dram_alt", imx8mq_dram_alt_sels, base + 0xa000, CLK_GET_RATE_NOCACHE); + hws[IMX8MQ_CLK_DRAM_APB] = __imx8m_clk_hw_composite("dram_apb", imx8mq_dram_apb_sels, base + 0xa080, CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); /* IP */ - clks[IMX8MQ_CLK_DRAM_CORE] = imx_clk_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mq_dram_core_sels, ARRAY_SIZE(imx8mq_dram_core_sels), CLK_IS_CRITICAL); - - clks[IMX8MQ_CLK_DRAM_ALT] = imx8m_clk_composite("dram_alt", imx8mq_dram_alt_sels, base + 0xa000); - clks[IMX8MQ_CLK_DRAM_APB] = imx8m_clk_composite_critical("dram_apb", imx8mq_dram_apb_sels, base + 0xa080); - clks[IMX8MQ_CLK_VPU_G1] = imx8m_clk_composite("vpu_g1", imx8mq_vpu_g1_sels, base + 0xa100); - clks[IMX8MQ_CLK_VPU_G2] = imx8m_clk_composite("vpu_g2", imx8mq_vpu_g2_sels, base + 0xa180); - clks[IMX8MQ_CLK_DISP_DTRC] = imx8m_clk_composite("disp_dtrc", imx8mq_disp_dtrc_sels, base + 0xa200); - clks[IMX8MQ_CLK_DISP_DC8000] = imx8m_clk_composite("disp_dc8000", imx8mq_disp_dc8000_sels, base + 0xa280); - clks[IMX8MQ_CLK_PCIE1_CTRL] = imx8m_clk_composite("pcie1_ctrl", imx8mq_pcie1_ctrl_sels, base + 0xa300); - clks[IMX8MQ_CLK_PCIE1_PHY] = imx8m_clk_composite("pcie1_phy", imx8mq_pcie1_phy_sels, base + 0xa380); - clks[IMX8MQ_CLK_PCIE1_AUX] = imx8m_clk_composite("pcie1_aux", imx8mq_pcie1_aux_sels, base + 0xa400); - clks[IMX8MQ_CLK_DC_PIXEL] = imx8m_clk_composite("dc_pixel", imx8mq_dc_pixel_sels, base + 0xa480); - clks[IMX8MQ_CLK_LCDIF_PIXEL] = imx8m_clk_composite("lcdif_pixel", imx8mq_lcdif_pixel_sels, base + 0xa500); - clks[IMX8MQ_CLK_SAI1] = imx8m_clk_composite("sai1", imx8mq_sai1_sels, base + 0xa580); - clks[IMX8MQ_CLK_SAI2] = imx8m_clk_composite("sai2", imx8mq_sai2_sels, base + 0xa600); - clks[IMX8MQ_CLK_SAI3] = imx8m_clk_composite("sai3", imx8mq_sai3_sels, base + 0xa680); - clks[IMX8MQ_CLK_SAI4] = imx8m_clk_composite("sai4", imx8mq_sai4_sels, base + 0xa700); - clks[IMX8MQ_CLK_SAI5] = imx8m_clk_composite("sai5", imx8mq_sai5_sels, base + 0xa780); - clks[IMX8MQ_CLK_SAI6] = imx8m_clk_composite("sai6", imx8mq_sai6_sels, base + 0xa800); - clks[IMX8MQ_CLK_SPDIF1] = imx8m_clk_composite("spdif1", imx8mq_spdif1_sels, base + 0xa880); - clks[IMX8MQ_CLK_SPDIF2] = imx8m_clk_composite("spdif2", imx8mq_spdif2_sels, base + 0xa900); - clks[IMX8MQ_CLK_ENET_REF] = imx8m_clk_composite("enet_ref", imx8mq_enet_ref_sels, base + 0xa980); - clks[IMX8MQ_CLK_ENET_TIMER] = imx8m_clk_composite("enet_timer", imx8mq_enet_timer_sels, base + 0xaa00); - clks[IMX8MQ_CLK_ENET_PHY_REF] = imx8m_clk_composite("enet_phy", imx8mq_enet_phy_sels, base + 0xaa80); - clks[IMX8MQ_CLK_NAND] = imx8m_clk_composite("nand", imx8mq_nand_sels, base + 0xab00); - clks[IMX8MQ_CLK_QSPI] = imx8m_clk_composite("qspi", imx8mq_qspi_sels, base + 0xab80); - clks[IMX8MQ_CLK_USDHC1] = imx8m_clk_composite("usdhc1", imx8mq_usdhc1_sels, base + 0xac00); - clks[IMX8MQ_CLK_USDHC2] = imx8m_clk_composite("usdhc2", imx8mq_usdhc2_sels, base + 0xac80); - clks[IMX8MQ_CLK_I2C1] = imx8m_clk_composite("i2c1", imx8mq_i2c1_sels, base + 0xad00); - clks[IMX8MQ_CLK_I2C2] = imx8m_clk_composite("i2c2", imx8mq_i2c2_sels, base + 0xad80); - clks[IMX8MQ_CLK_I2C3] = imx8m_clk_composite("i2c3", imx8mq_i2c3_sels, base + 0xae00); - clks[IMX8MQ_CLK_I2C4] = imx8m_clk_composite("i2c4", imx8mq_i2c4_sels, base + 0xae80); - clks[IMX8MQ_CLK_UART1] = imx8m_clk_composite("uart1", imx8mq_uart1_sels, base + 0xaf00); - clks[IMX8MQ_CLK_UART2] = imx8m_clk_composite("uart2", imx8mq_uart2_sels, base + 0xaf80); - clks[IMX8MQ_CLK_UART3] = imx8m_clk_composite("uart3", imx8mq_uart3_sels, base + 0xb000); - clks[IMX8MQ_CLK_UART4] = imx8m_clk_composite("uart4", imx8mq_uart4_sels, base + 0xb080); - clks[IMX8MQ_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mq_usb_core_sels, base + 0xb100); - clks[IMX8MQ_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mq_usb_phy_sels, base + 0xb180); - clks[IMX8MQ_CLK_GIC] = imx8m_clk_composite_critical("gic", imx8mq_gic_sels, base + 0xb200); - clks[IMX8MQ_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mq_ecspi1_sels, base + 0xb280); - clks[IMX8MQ_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mq_ecspi2_sels, base + 0xb300); - clks[IMX8MQ_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mq_pwm1_sels, base + 0xb380); - clks[IMX8MQ_CLK_PWM2] = imx8m_clk_composite("pwm2", imx8mq_pwm2_sels, base + 0xb400); - clks[IMX8MQ_CLK_PWM3] = imx8m_clk_composite("pwm3", imx8mq_pwm3_sels, base + 0xb480); - clks[IMX8MQ_CLK_PWM4] = imx8m_clk_composite("pwm4", imx8mq_pwm4_sels, base + 0xb500); - clks[IMX8MQ_CLK_GPT1] = imx8m_clk_composite("gpt1", imx8mq_gpt1_sels, base + 0xb580); - clks[IMX8MQ_CLK_WDOG] = imx8m_clk_composite("wdog", imx8mq_wdog_sels, base + 0xb900); - clks[IMX8MQ_CLK_WRCLK] = imx8m_clk_composite("wrclk", imx8mq_wrclk_sels, base + 0xb980); - clks[IMX8MQ_CLK_CLKO1] = imx8m_clk_composite("clko1", imx8mq_clko1_sels, base + 0xba00); - clks[IMX8MQ_CLK_CLKO2] = imx8m_clk_composite("clko2", imx8mq_clko2_sels, base + 0xba80); - clks[IMX8MQ_CLK_DSI_CORE] = imx8m_clk_composite("dsi_core", imx8mq_dsi_core_sels, base + 0xbb00); - clks[IMX8MQ_CLK_DSI_PHY_REF] = imx8m_clk_composite("dsi_phy_ref", imx8mq_dsi_phy_sels, base + 0xbb80); - clks[IMX8MQ_CLK_DSI_DBI] = imx8m_clk_composite("dsi_dbi", imx8mq_dsi_dbi_sels, base + 0xbc00); - clks[IMX8MQ_CLK_DSI_ESC] = imx8m_clk_composite("dsi_esc", imx8mq_dsi_esc_sels, base + 0xbc80); - clks[IMX8MQ_CLK_DSI_AHB] = imx8m_clk_composite("dsi_ahb", imx8mq_dsi_ahb_sels, base + 0x9200); - clks[IMX8MQ_CLK_DSI_IPG_DIV] = imx_clk_divider2("dsi_ipg_div", "dsi_ahb", base + 0x9280, 0, 6); - clks[IMX8MQ_CLK_CSI1_CORE] = imx8m_clk_composite("csi1_core", imx8mq_csi1_core_sels, base + 0xbd00); - clks[IMX8MQ_CLK_CSI1_PHY_REF] = imx8m_clk_composite("csi1_phy_ref", imx8mq_csi1_phy_sels, base + 0xbd80); - clks[IMX8MQ_CLK_CSI1_ESC] = imx8m_clk_composite("csi1_esc", imx8mq_csi1_esc_sels, base + 0xbe00); - clks[IMX8MQ_CLK_CSI2_CORE] = imx8m_clk_composite("csi2_core", imx8mq_csi2_core_sels, base + 0xbe80); - clks[IMX8MQ_CLK_CSI2_PHY_REF] = imx8m_clk_composite("csi2_phy_ref", imx8mq_csi2_phy_sels, base + 0xbf00); - clks[IMX8MQ_CLK_CSI2_ESC] = imx8m_clk_composite("csi2_esc", imx8mq_csi2_esc_sels, base + 0xbf80); - clks[IMX8MQ_CLK_PCIE2_CTRL] = imx8m_clk_composite("pcie2_ctrl", imx8mq_pcie2_ctrl_sels, base + 0xc000); - clks[IMX8MQ_CLK_PCIE2_PHY] = imx8m_clk_composite("pcie2_phy", imx8mq_pcie2_phy_sels, base + 0xc080); - clks[IMX8MQ_CLK_PCIE2_AUX] = imx8m_clk_composite("pcie2_aux", imx8mq_pcie2_aux_sels, base + 0xc100); - clks[IMX8MQ_CLK_ECSPI3] = imx8m_clk_composite("ecspi3", imx8mq_ecspi3_sels, base + 0xc180); - - clks[IMX8MQ_CLK_ECSPI1_ROOT] = imx_clk_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0); - clks[IMX8MQ_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); - clks[IMX8MQ_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); - clks[IMX8MQ_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); - clks[IMX8MQ_CLK_GPIO1_ROOT] = imx_clk_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); - clks[IMX8MQ_CLK_GPIO2_ROOT] = imx_clk_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); - clks[IMX8MQ_CLK_GPIO3_ROOT] = imx_clk_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); - clks[IMX8MQ_CLK_GPIO4_ROOT] = imx_clk_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); - clks[IMX8MQ_CLK_GPIO5_ROOT] = imx_clk_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); - clks[IMX8MQ_CLK_GPT1_ROOT] = imx_clk_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0); - clks[IMX8MQ_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); - clks[IMX8MQ_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); - clks[IMX8MQ_CLK_I2C3_ROOT] = imx_clk_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0); - clks[IMX8MQ_CLK_I2C4_ROOT] = imx_clk_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0); - clks[IMX8MQ_CLK_MU_ROOT] = imx_clk_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0); - clks[IMX8MQ_CLK_OCOTP_ROOT] = imx_clk_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0); - clks[IMX8MQ_CLK_PCIE1_ROOT] = imx_clk_gate4("pcie1_root_clk", "pcie1_ctrl", base + 0x4250, 0); - clks[IMX8MQ_CLK_PCIE2_ROOT] = imx_clk_gate4("pcie2_root_clk", "pcie2_ctrl", base + 0x4640, 0); - clks[IMX8MQ_CLK_PWM1_ROOT] = imx_clk_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0); - clks[IMX8MQ_CLK_PWM2_ROOT] = imx_clk_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0); - clks[IMX8MQ_CLK_PWM3_ROOT] = imx_clk_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0); - clks[IMX8MQ_CLK_PWM4_ROOT] = imx_clk_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0); - clks[IMX8MQ_CLK_QSPI_ROOT] = imx_clk_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0); - clks[IMX8MQ_CLK_RAWNAND_ROOT] = imx_clk_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand); - clks[IMX8MQ_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand); - clks[IMX8MQ_CLK_SAI1_ROOT] = imx_clk_gate2_shared2("sai1_root_clk", "sai1", base + 0x4330, 0, &share_count_sai1); - clks[IMX8MQ_CLK_SAI1_IPG] = imx_clk_gate2_shared2("sai1_ipg_clk", "ipg_audio_root", base + 0x4330, 0, &share_count_sai1); - clks[IMX8MQ_CLK_SAI2_ROOT] = imx_clk_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2); - clks[IMX8MQ_CLK_SAI2_IPG] = imx_clk_gate2_shared2("sai2_ipg_clk", "ipg_root", base + 0x4340, 0, &share_count_sai2); - clks[IMX8MQ_CLK_SAI3_ROOT] = imx_clk_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3); - clks[IMX8MQ_CLK_SAI3_IPG] = imx_clk_gate2_shared2("sai3_ipg_clk", "ipg_root", base + 0x4350, 0, &share_count_sai3); - clks[IMX8MQ_CLK_SAI4_ROOT] = imx_clk_gate2_shared2("sai4_root_clk", "sai4", base + 0x4360, 0, &share_count_sai4); - clks[IMX8MQ_CLK_SAI4_IPG] = imx_clk_gate2_shared2("sai4_ipg_clk", "ipg_audio_root", base + 0x4360, 0, &share_count_sai4); - clks[IMX8MQ_CLK_SAI5_ROOT] = imx_clk_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5); - clks[IMX8MQ_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); - clks[IMX8MQ_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); - clks[IMX8MQ_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); - clks[IMX8MQ_CLK_SNVS_ROOT] = imx_clk_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0); - clks[IMX8MQ_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); - clks[IMX8MQ_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); - clks[IMX8MQ_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); - clks[IMX8MQ_CLK_UART4_ROOT] = imx_clk_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0); - clks[IMX8MQ_CLK_USB1_CTRL_ROOT] = imx_clk_gate4("usb1_ctrl_root_clk", "usb_bus", base + 0x44d0, 0); - clks[IMX8MQ_CLK_USB2_CTRL_ROOT] = imx_clk_gate4("usb2_ctrl_root_clk", "usb_bus", base + 0x44e0, 0); - clks[IMX8MQ_CLK_USB1_PHY_ROOT] = imx_clk_gate4("usb1_phy_root_clk", "usb_phy_ref", base + 0x44f0, 0); - clks[IMX8MQ_CLK_USB2_PHY_ROOT] = imx_clk_gate4("usb2_phy_root_clk", "usb_phy_ref", base + 0x4500, 0); - clks[IMX8MQ_CLK_USDHC1_ROOT] = imx_clk_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0); - clks[IMX8MQ_CLK_USDHC2_ROOT] = imx_clk_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0); - clks[IMX8MQ_CLK_WDOG1_ROOT] = imx_clk_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0); - clks[IMX8MQ_CLK_WDOG2_ROOT] = imx_clk_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0); - clks[IMX8MQ_CLK_WDOG3_ROOT] = imx_clk_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0); - clks[IMX8MQ_CLK_VPU_G1_ROOT] = imx_clk_gate2_flags("vpu_g1_root_clk", "vpu_g1", base + 0x4560, 0, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); - clks[IMX8MQ_CLK_GPU_ROOT] = imx_clk_gate4("gpu_root_clk", "gpu_core_div", base + 0x4570, 0); - clks[IMX8MQ_CLK_VPU_G2_ROOT] = imx_clk_gate2_flags("vpu_g2_root_clk", "vpu_g2", base + 0x45a0, 0, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); - clks[IMX8MQ_CLK_DISP_ROOT] = imx_clk_gate2_shared2("disp_root_clk", "disp_dc8000", base + 0x45d0, 0, &share_count_dcss); - clks[IMX8MQ_CLK_DISP_AXI_ROOT] = imx_clk_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_dcss); - clks[IMX8MQ_CLK_DISP_APB_ROOT] = imx_clk_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_dcss); - clks[IMX8MQ_CLK_DISP_RTRM_ROOT] = imx_clk_gate2_shared2("disp_rtrm_root_clk", "disp_rtrm", base + 0x45d0, 0, &share_count_dcss); - clks[IMX8MQ_CLK_TMU_ROOT] = imx_clk_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0); - clks[IMX8MQ_CLK_VPU_DEC_ROOT] = imx_clk_gate2_flags("vpu_dec_root_clk", "vpu_bus", base + 0x4630, 0, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); - clks[IMX8MQ_CLK_CSI1_ROOT] = imx_clk_gate4("csi1_root_clk", "csi1_core", base + 0x4650, 0); - clks[IMX8MQ_CLK_CSI2_ROOT] = imx_clk_gate4("csi2_root_clk", "csi2_core", base + 0x4660, 0); - clks[IMX8MQ_CLK_SDMA1_ROOT] = imx_clk_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0); - clks[IMX8MQ_CLK_SDMA2_ROOT] = imx_clk_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0); - - clks[IMX8MQ_GPT_3M_CLK] = imx_clk_fixed_factor("gpt_3m", "osc_25m", 1, 8); - clks[IMX8MQ_CLK_DRAM_ALT_ROOT] = imx_clk_fixed_factor("dram_alt_root", "dram_alt", 1, 4); - - clks[IMX8MQ_CLK_ARM] = imx_clk_cpu("arm", "arm_a53_div", - clks[IMX8MQ_CLK_A53_DIV], - clks[IMX8MQ_CLK_A53_SRC], - clks[IMX8MQ_ARM_PLL_OUT], - clks[IMX8MQ_SYS1_PLL_800M]); - - imx_check_clocks(clks, ARRAY_SIZE(clks)); - - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - - err = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + hws[IMX8MQ_CLK_VPU_G1] = imx8m_clk_hw_composite("vpu_g1", imx8mq_vpu_g1_sels, base + 0xa100); + hws[IMX8MQ_CLK_VPU_G2] = imx8m_clk_hw_composite("vpu_g2", imx8mq_vpu_g2_sels, base + 0xa180); + hws[IMX8MQ_CLK_DISP_DTRC] = imx8m_clk_hw_composite("disp_dtrc", imx8mq_disp_dtrc_sels, base + 0xa200); + hws[IMX8MQ_CLK_DISP_DC8000] = imx8m_clk_hw_composite("disp_dc8000", imx8mq_disp_dc8000_sels, base + 0xa280); + hws[IMX8MQ_CLK_PCIE1_CTRL] = imx8m_clk_hw_composite("pcie1_ctrl", imx8mq_pcie1_ctrl_sels, base + 0xa300); + hws[IMX8MQ_CLK_PCIE1_PHY] = imx8m_clk_hw_composite("pcie1_phy", imx8mq_pcie1_phy_sels, base + 0xa380); + hws[IMX8MQ_CLK_PCIE1_AUX] = imx8m_clk_hw_composite("pcie1_aux", imx8mq_pcie1_aux_sels, base + 0xa400); + hws[IMX8MQ_CLK_DC_PIXEL] = imx8m_clk_hw_composite("dc_pixel", imx8mq_dc_pixel_sels, base + 0xa480); + hws[IMX8MQ_CLK_LCDIF_PIXEL] = imx8m_clk_hw_composite("lcdif_pixel", imx8mq_lcdif_pixel_sels, base + 0xa500); + hws[IMX8MQ_CLK_SAI1] = imx8m_clk_hw_composite("sai1", imx8mq_sai1_sels, base + 0xa580); + hws[IMX8MQ_CLK_SAI2] = imx8m_clk_hw_composite("sai2", imx8mq_sai2_sels, base + 0xa600); + hws[IMX8MQ_CLK_SAI3] = imx8m_clk_hw_composite("sai3", imx8mq_sai3_sels, base + 0xa680); + hws[IMX8MQ_CLK_SAI4] = imx8m_clk_hw_composite("sai4", imx8mq_sai4_sels, base + 0xa700); + hws[IMX8MQ_CLK_SAI5] = imx8m_clk_hw_composite("sai5", imx8mq_sai5_sels, base + 0xa780); + hws[IMX8MQ_CLK_SAI6] = imx8m_clk_hw_composite("sai6", imx8mq_sai6_sels, base + 0xa800); + hws[IMX8MQ_CLK_SPDIF1] = imx8m_clk_hw_composite("spdif1", imx8mq_spdif1_sels, base + 0xa880); + hws[IMX8MQ_CLK_SPDIF2] = imx8m_clk_hw_composite("spdif2", imx8mq_spdif2_sels, base + 0xa900); + hws[IMX8MQ_CLK_ENET_REF] = imx8m_clk_hw_composite("enet_ref", imx8mq_enet_ref_sels, base + 0xa980); + hws[IMX8MQ_CLK_ENET_TIMER] = imx8m_clk_hw_composite("enet_timer", imx8mq_enet_timer_sels, base + 0xaa00); + hws[IMX8MQ_CLK_ENET_PHY_REF] = imx8m_clk_hw_composite("enet_phy", imx8mq_enet_phy_sels, base + 0xaa80); + hws[IMX8MQ_CLK_NAND] = imx8m_clk_hw_composite("nand", imx8mq_nand_sels, base + 0xab00); + hws[IMX8MQ_CLK_QSPI] = imx8m_clk_hw_composite("qspi", imx8mq_qspi_sels, base + 0xab80); + hws[IMX8MQ_CLK_USDHC1] = imx8m_clk_hw_composite("usdhc1", imx8mq_usdhc1_sels, base + 0xac00); + hws[IMX8MQ_CLK_USDHC2] = imx8m_clk_hw_composite("usdhc2", imx8mq_usdhc2_sels, base + 0xac80); + hws[IMX8MQ_CLK_I2C1] = imx8m_clk_hw_composite("i2c1", imx8mq_i2c1_sels, base + 0xad00); + hws[IMX8MQ_CLK_I2C2] = imx8m_clk_hw_composite("i2c2", imx8mq_i2c2_sels, base + 0xad80); + hws[IMX8MQ_CLK_I2C3] = imx8m_clk_hw_composite("i2c3", imx8mq_i2c3_sels, base + 0xae00); + hws[IMX8MQ_CLK_I2C4] = imx8m_clk_hw_composite("i2c4", imx8mq_i2c4_sels, base + 0xae80); + hws[IMX8MQ_CLK_UART1] = imx8m_clk_hw_composite("uart1", imx8mq_uart1_sels, base + 0xaf00); + hws[IMX8MQ_CLK_UART2] = imx8m_clk_hw_composite("uart2", imx8mq_uart2_sels, base + 0xaf80); + hws[IMX8MQ_CLK_UART3] = imx8m_clk_hw_composite("uart3", imx8mq_uart3_sels, base + 0xb000); + hws[IMX8MQ_CLK_UART4] = imx8m_clk_hw_composite("uart4", imx8mq_uart4_sels, base + 0xb080); + hws[IMX8MQ_CLK_USB_CORE_REF] = imx8m_clk_hw_composite("usb_core_ref", imx8mq_usb_core_sels, base + 0xb100); + hws[IMX8MQ_CLK_USB_PHY_REF] = imx8m_clk_hw_composite("usb_phy_ref", imx8mq_usb_phy_sels, base + 0xb180); + hws[IMX8MQ_CLK_GIC] = imx8m_clk_hw_composite_critical("gic", imx8mq_gic_sels, base + 0xb200); + hws[IMX8MQ_CLK_ECSPI1] = imx8m_clk_hw_composite("ecspi1", imx8mq_ecspi1_sels, base + 0xb280); + hws[IMX8MQ_CLK_ECSPI2] = imx8m_clk_hw_composite("ecspi2", imx8mq_ecspi2_sels, base + 0xb300); + hws[IMX8MQ_CLK_PWM1] = imx8m_clk_hw_composite("pwm1", imx8mq_pwm1_sels, base + 0xb380); + hws[IMX8MQ_CLK_PWM2] = imx8m_clk_hw_composite("pwm2", imx8mq_pwm2_sels, base + 0xb400); + hws[IMX8MQ_CLK_PWM3] = imx8m_clk_hw_composite("pwm3", imx8mq_pwm3_sels, base + 0xb480); + hws[IMX8MQ_CLK_PWM4] = imx8m_clk_hw_composite("pwm4", imx8mq_pwm4_sels, base + 0xb500); + hws[IMX8MQ_CLK_GPT1] = imx8m_clk_hw_composite("gpt1", imx8mq_gpt1_sels, base + 0xb580); + hws[IMX8MQ_CLK_WDOG] = imx8m_clk_hw_composite("wdog", imx8mq_wdog_sels, base + 0xb900); + hws[IMX8MQ_CLK_WRCLK] = imx8m_clk_hw_composite("wrclk", imx8mq_wrclk_sels, base + 0xb980); + hws[IMX8MQ_CLK_CLKO1] = imx8m_clk_hw_composite("clko1", imx8mq_clko1_sels, base + 0xba00); + hws[IMX8MQ_CLK_CLKO2] = imx8m_clk_hw_composite("clko2", imx8mq_clko2_sels, base + 0xba80); + hws[IMX8MQ_CLK_DSI_CORE] = imx8m_clk_hw_composite("dsi_core", imx8mq_dsi_core_sels, base + 0xbb00); + hws[IMX8MQ_CLK_DSI_PHY_REF] = imx8m_clk_hw_composite("dsi_phy_ref", imx8mq_dsi_phy_sels, base + 0xbb80); + hws[IMX8MQ_CLK_DSI_DBI] = imx8m_clk_hw_composite("dsi_dbi", imx8mq_dsi_dbi_sels, base + 0xbc00); + hws[IMX8MQ_CLK_DSI_ESC] = imx8m_clk_hw_composite("dsi_esc", imx8mq_dsi_esc_sels, base + 0xbc80); + hws[IMX8MQ_CLK_DSI_AHB] = imx8m_clk_hw_composite("dsi_ahb", imx8mq_dsi_ahb_sels, base + 0x9200); + hws[IMX8MQ_CLK_DSI_IPG_DIV] = imx_clk_hw_divider2("dsi_ipg_div", "dsi_ahb", base + 0x9280, 0, 6); + hws[IMX8MQ_CLK_CSI1_CORE] = imx8m_clk_hw_composite("csi1_core", imx8mq_csi1_core_sels, base + 0xbd00); + hws[IMX8MQ_CLK_CSI1_PHY_REF] = imx8m_clk_hw_composite("csi1_phy_ref", imx8mq_csi1_phy_sels, base + 0xbd80); + hws[IMX8MQ_CLK_CSI1_ESC] = imx8m_clk_hw_composite("csi1_esc", imx8mq_csi1_esc_sels, base + 0xbe00); + hws[IMX8MQ_CLK_CSI2_CORE] = imx8m_clk_hw_composite("csi2_core", imx8mq_csi2_core_sels, base + 0xbe80); + hws[IMX8MQ_CLK_CSI2_PHY_REF] = imx8m_clk_hw_composite("csi2_phy_ref", imx8mq_csi2_phy_sels, base + 0xbf00); + hws[IMX8MQ_CLK_CSI2_ESC] = imx8m_clk_hw_composite("csi2_esc", imx8mq_csi2_esc_sels, base + 0xbf80); + hws[IMX8MQ_CLK_PCIE2_CTRL] = imx8m_clk_hw_composite("pcie2_ctrl", imx8mq_pcie2_ctrl_sels, base + 0xc000); + hws[IMX8MQ_CLK_PCIE2_PHY] = imx8m_clk_hw_composite("pcie2_phy", imx8mq_pcie2_phy_sels, base + 0xc080); + hws[IMX8MQ_CLK_PCIE2_AUX] = imx8m_clk_hw_composite("pcie2_aux", imx8mq_pcie2_aux_sels, base + 0xc100); + hws[IMX8MQ_CLK_ECSPI3] = imx8m_clk_hw_composite("ecspi3", imx8mq_ecspi3_sels, base + 0xc180); + + hws[IMX8MQ_CLK_ECSPI1_ROOT] = imx_clk_hw_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0); + hws[IMX8MQ_CLK_ECSPI2_ROOT] = imx_clk_hw_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); + hws[IMX8MQ_CLK_ECSPI3_ROOT] = imx_clk_hw_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); + hws[IMX8MQ_CLK_ENET1_ROOT] = imx_clk_hw_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); + hws[IMX8MQ_CLK_GPIO1_ROOT] = imx_clk_hw_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); + hws[IMX8MQ_CLK_GPIO2_ROOT] = imx_clk_hw_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); + hws[IMX8MQ_CLK_GPIO3_ROOT] = imx_clk_hw_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); + hws[IMX8MQ_CLK_GPIO4_ROOT] = imx_clk_hw_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); + hws[IMX8MQ_CLK_GPIO5_ROOT] = imx_clk_hw_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); + hws[IMX8MQ_CLK_GPT1_ROOT] = imx_clk_hw_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0); + hws[IMX8MQ_CLK_I2C1_ROOT] = imx_clk_hw_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); + hws[IMX8MQ_CLK_I2C2_ROOT] = imx_clk_hw_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); + hws[IMX8MQ_CLK_I2C3_ROOT] = imx_clk_hw_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0); + hws[IMX8MQ_CLK_I2C4_ROOT] = imx_clk_hw_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0); + hws[IMX8MQ_CLK_MU_ROOT] = imx_clk_hw_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0); + hws[IMX8MQ_CLK_OCOTP_ROOT] = imx_clk_hw_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0); + hws[IMX8MQ_CLK_PCIE1_ROOT] = imx_clk_hw_gate4("pcie1_root_clk", "pcie1_ctrl", base + 0x4250, 0); + hws[IMX8MQ_CLK_PCIE2_ROOT] = imx_clk_hw_gate4("pcie2_root_clk", "pcie2_ctrl", base + 0x4640, 0); + hws[IMX8MQ_CLK_PWM1_ROOT] = imx_clk_hw_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0); + hws[IMX8MQ_CLK_PWM2_ROOT] = imx_clk_hw_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0); + hws[IMX8MQ_CLK_PWM3_ROOT] = imx_clk_hw_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0); + hws[IMX8MQ_CLK_PWM4_ROOT] = imx_clk_hw_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0); + hws[IMX8MQ_CLK_QSPI_ROOT] = imx_clk_hw_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0); + hws[IMX8MQ_CLK_RAWNAND_ROOT] = imx_clk_hw_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand); + hws[IMX8MQ_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand); + hws[IMX8MQ_CLK_SAI1_ROOT] = imx_clk_hw_gate2_shared2("sai1_root_clk", "sai1", base + 0x4330, 0, &share_count_sai1); + hws[IMX8MQ_CLK_SAI1_IPG] = imx_clk_hw_gate2_shared2("sai1_ipg_clk", "ipg_audio_root", base + 0x4330, 0, &share_count_sai1); + hws[IMX8MQ_CLK_SAI2_ROOT] = imx_clk_hw_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2); + hws[IMX8MQ_CLK_SAI2_IPG] = imx_clk_hw_gate2_shared2("sai2_ipg_clk", "ipg_root", base + 0x4340, 0, &share_count_sai2); + hws[IMX8MQ_CLK_SAI3_ROOT] = imx_clk_hw_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3); + hws[IMX8MQ_CLK_SAI3_IPG] = imx_clk_hw_gate2_shared2("sai3_ipg_clk", "ipg_root", base + 0x4350, 0, &share_count_sai3); + hws[IMX8MQ_CLK_SAI4_ROOT] = imx_clk_hw_gate2_shared2("sai4_root_clk", "sai4", base + 0x4360, 0, &share_count_sai4); + hws[IMX8MQ_CLK_SAI4_IPG] = imx_clk_hw_gate2_shared2("sai4_ipg_clk", "ipg_audio_root", base + 0x4360, 0, &share_count_sai4); + hws[IMX8MQ_CLK_SAI5_ROOT] = imx_clk_hw_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5); + hws[IMX8MQ_CLK_SAI5_IPG] = imx_clk_hw_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); + hws[IMX8MQ_CLK_SAI6_ROOT] = imx_clk_hw_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); + hws[IMX8MQ_CLK_SAI6_IPG] = imx_clk_hw_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); + hws[IMX8MQ_CLK_SNVS_ROOT] = imx_clk_hw_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0); + hws[IMX8MQ_CLK_UART1_ROOT] = imx_clk_hw_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); + hws[IMX8MQ_CLK_UART2_ROOT] = imx_clk_hw_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); + hws[IMX8MQ_CLK_UART3_ROOT] = imx_clk_hw_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); + hws[IMX8MQ_CLK_UART4_ROOT] = imx_clk_hw_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0); + hws[IMX8MQ_CLK_USB1_CTRL_ROOT] = imx_clk_hw_gate4("usb1_ctrl_root_clk", "usb_bus", base + 0x44d0, 0); + hws[IMX8MQ_CLK_USB2_CTRL_ROOT] = imx_clk_hw_gate4("usb2_ctrl_root_clk", "usb_bus", base + 0x44e0, 0); + hws[IMX8MQ_CLK_USB1_PHY_ROOT] = imx_clk_hw_gate4("usb1_phy_root_clk", "usb_phy_ref", base + 0x44f0, 0); + hws[IMX8MQ_CLK_USB2_PHY_ROOT] = imx_clk_hw_gate4("usb2_phy_root_clk", "usb_phy_ref", base + 0x4500, 0); + hws[IMX8MQ_CLK_USDHC1_ROOT] = imx_clk_hw_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0); + hws[IMX8MQ_CLK_USDHC2_ROOT] = imx_clk_hw_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0); + hws[IMX8MQ_CLK_WDOG1_ROOT] = imx_clk_hw_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0); + hws[IMX8MQ_CLK_WDOG2_ROOT] = imx_clk_hw_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0); + hws[IMX8MQ_CLK_WDOG3_ROOT] = imx_clk_hw_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0); + hws[IMX8MQ_CLK_VPU_G1_ROOT] = imx_clk_hw_gate2_flags("vpu_g1_root_clk", "vpu_g1", base + 0x4560, 0, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); + hws[IMX8MQ_CLK_GPU_ROOT] = imx_clk_hw_gate4("gpu_root_clk", "gpu_core_div", base + 0x4570, 0); + hws[IMX8MQ_CLK_VPU_G2_ROOT] = imx_clk_hw_gate2_flags("vpu_g2_root_clk", "vpu_g2", base + 0x45a0, 0, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); + hws[IMX8MQ_CLK_DISP_ROOT] = imx_clk_hw_gate2_shared2("disp_root_clk", "disp_dc8000", base + 0x45d0, 0, &share_count_dcss); + hws[IMX8MQ_CLK_DISP_AXI_ROOT] = imx_clk_hw_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_dcss); + hws[IMX8MQ_CLK_DISP_APB_ROOT] = imx_clk_hw_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_dcss); + hws[IMX8MQ_CLK_DISP_RTRM_ROOT] = imx_clk_hw_gate2_shared2("disp_rtrm_root_clk", "disp_rtrm", base + 0x45d0, 0, &share_count_dcss); + hws[IMX8MQ_CLK_TMU_ROOT] = imx_clk_hw_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0); + hws[IMX8MQ_CLK_VPU_DEC_ROOT] = imx_clk_hw_gate2_flags("vpu_dec_root_clk", "vpu_bus", base + 0x4630, 0, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE); + hws[IMX8MQ_CLK_CSI1_ROOT] = imx_clk_hw_gate4("csi1_root_clk", "csi1_core", base + 0x4650, 0); + hws[IMX8MQ_CLK_CSI2_ROOT] = imx_clk_hw_gate4("csi2_root_clk", "csi2_core", base + 0x4660, 0); + hws[IMX8MQ_CLK_SDMA1_ROOT] = imx_clk_hw_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0); + hws[IMX8MQ_CLK_SDMA2_ROOT] = imx_clk_hw_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0); + + hws[IMX8MQ_GPT_3M_CLK] = imx_clk_hw_fixed_factor("gpt_3m", "osc_25m", 1, 8); + hws[IMX8MQ_CLK_DRAM_ALT_ROOT] = imx_clk_hw_fixed_factor("dram_alt_root", "dram_alt", 1, 4); + + hws[IMX8MQ_CLK_ARM] = imx_clk_hw_cpu("arm", "arm_a53_div", + hws[IMX8MQ_CLK_A53_DIV]->clk, + hws[IMX8MQ_CLK_A53_SRC]->clk, + hws[IMX8MQ_ARM_PLL_OUT]->clk, + hws[IMX8MQ_SYS1_PLL_800M]->clk); + + imx_check_clk_hws(hws, IMX8MQ_CLK_END); + + err = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); if (err < 0) { - dev_err(dev, "failed to register clks for i.MX8MQ\n"); - goto unregister_clks; + dev_err(dev, "failed to register hws for i.MX8MQ\n"); + goto unregister_hws; + } + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_hws[i] = &hws[index]->clk; } - imx_register_uart_clocks(uart_clks); + imx_register_uart_clocks(uart_hws); return 0; -unregister_clks: - imx_unregister_clocks(clks, ARRAY_SIZE(clks)); +unregister_hws: + imx_unregister_hw_clocks(hws, IMX8MQ_CLK_END); return err; } @@ -609,6 +624,11 @@ static struct platform_driver imx8mq_clk_driver = { .probe = imx8mq_clocks_probe, .driver = { .name = "imx8mq-ccm", + /* + * Disable bind attributes: clocks are not removed and + * reloading the driver will crash or break devices. + */ + .suppress_bind_attrs = true, .of_match_table = of_match_ptr(imx8mq_clk_of_match), }, }; diff --git a/drivers/clk/imx/clk-imx8qxp-lpcg.c b/drivers/clk/imx/clk-imx8qxp-lpcg.c index c0aff7ca6374..04c8ee35e14c 100644 --- a/drivers/clk/imx/clk-imx8qxp-lpcg.c +++ b/drivers/clk/imx/clk-imx8qxp-lpcg.c @@ -173,6 +173,17 @@ static int imx8qxp_lpcg_clk_probe(struct platform_device *pdev) if (!ss_lpcg) return -ENODEV; + /* + * Please don't replace this with devm_platform_ioremap_resource. + * + * devm_platform_ioremap_resource calls devm_ioremap_resource which + * differs from devm_ioremap by also calling devm_request_mem_region + * and preventing other mappings in the same area. + * + * On imx8 the LPCG nodes map entire subsystems and overlap + * peripherals, this means that using devm_platform_ioremap_resource + * will cause many devices to fail to probe including serial ports. + */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) return -EINVAL; diff --git a/drivers/clk/imx/clk-pfdv2.c b/drivers/clk/imx/clk-pfdv2.c index a03bbed662c6..de93ce73101b 100644 --- a/drivers/clk/imx/clk-pfdv2.c +++ b/drivers/clk/imx/clk-pfdv2.c @@ -166,7 +166,7 @@ static const struct clk_ops clk_pfdv2_ops = { .is_enabled = clk_pfdv2_is_enabled, }; -struct clk_hw *imx_clk_pfdv2(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_pfdv2(const char *name, const char *parent_name, void __iomem *reg, u8 idx) { struct clk_init_data init; diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c index 3636c8035c7d..5b0519a81a7a 100644 --- a/drivers/clk/imx/clk-pll14xx.c +++ b/drivers/clk/imx/clk-pll14xx.c @@ -67,6 +67,13 @@ struct imx_pll14xx_clk imx_1443x_pll = { .rate_count = ARRAY_SIZE(imx_pll1443x_tbl), }; +struct imx_pll14xx_clk imx_1443x_dram_pll = { + .type = PLL_1443X, + .rate_table = imx_pll1443x_tbl, + .rate_count = ARRAY_SIZE(imx_pll1443x_tbl), + .flags = CLK_GET_RATE_NOCACHE, +}; + struct imx_pll14xx_clk imx_1416x_pll = { .type = PLL_1416X, .rate_table = imx_pll1416x_tbl, @@ -369,13 +376,14 @@ static const struct clk_ops clk_pll1443x_ops = { .set_rate = clk_pll1443x_set_rate, }; -struct clk *imx_clk_pll14xx(const char *name, const char *parent_name, - void __iomem *base, - const struct imx_pll14xx_clk *pll_clk) +struct clk_hw *imx_clk_hw_pll14xx(const char *name, const char *parent_name, + void __iomem *base, + const struct imx_pll14xx_clk *pll_clk) { struct clk_pll14xx *pll; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; u32 val; pll = kzalloc(sizeof(*pll), GFP_KERNEL); @@ -412,12 +420,15 @@ struct clk *imx_clk_pll14xx(const char *name, const char *parent_name, val &= ~BYPASS_MASK; writel_relaxed(val, pll->base + GNRL_CTL); - clk = clk_register(NULL, &pll->hw); - if (IS_ERR(clk)) { - pr_err("%s: failed to register pll %s %lu\n", - __func__, name, PTR_ERR(clk)); + hw = &pll->hw; + + ret = clk_hw_register(NULL, hw); + if (ret) { + pr_err("%s: failed to register pll %s %d\n", + __func__, name, ret); kfree(pll); + return ERR_PTR(ret); } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-pllv1.c b/drivers/clk/imx/clk-pllv1.c index 4ba9973d4c18..de4f8a41a7d0 100644 --- a/drivers/clk/imx/clk-pllv1.c +++ b/drivers/clk/imx/clk-pllv1.c @@ -111,12 +111,13 @@ static const struct clk_ops clk_pllv1_ops = { .recalc_rate = clk_pllv1_recalc_rate, }; -struct clk *imx_clk_pllv1(enum imx_pllv1_type type, const char *name, +struct clk_hw *imx_clk_hw_pllv1(enum imx_pllv1_type type, const char *name, const char *parent, void __iomem *base) { struct clk_pllv1 *pll; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; pll = kmalloc(sizeof(*pll), GFP_KERNEL); if (!pll) @@ -132,10 +133,13 @@ struct clk *imx_clk_pllv1(enum imx_pllv1_type type, const char *name, init.num_parents = 1; pll->hw.init = &init; + hw = &pll->hw; - clk = clk_register(NULL, &pll->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(pll); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-pllv2.c b/drivers/clk/imx/clk-pllv2.c index eeba3cb14e2d..ff17f0664faa 100644 --- a/drivers/clk/imx/clk-pllv2.c +++ b/drivers/clk/imx/clk-pllv2.c @@ -239,12 +239,13 @@ static const struct clk_ops clk_pllv2_ops = { .set_rate = clk_pllv2_set_rate, }; -struct clk *imx_clk_pllv2(const char *name, const char *parent, +struct clk_hw *imx_clk_hw_pllv2(const char *name, const char *parent, void __iomem *base) { struct clk_pllv2 *pll; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; pll = kzalloc(sizeof(*pll), GFP_KERNEL); if (!pll) @@ -259,10 +260,13 @@ struct clk *imx_clk_pllv2(const char *name, const char *parent, init.num_parents = 1; pll->hw.init = &init; + hw = &pll->hw; - clk = clk_register(NULL, &pll->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(pll); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-pllv4.c b/drivers/clk/imx/clk-pllv4.c index 8155b12cf0e1..f51a800c268c 100644 --- a/drivers/clk/imx/clk-pllv4.c +++ b/drivers/clk/imx/clk-pllv4.c @@ -206,7 +206,7 @@ static const struct clk_ops clk_pllv4_ops = { .is_enabled = clk_pllv4_is_enabled, }; -struct clk_hw *imx_clk_pllv4(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_pllv4(const char *name, const char *parent_name, void __iomem *base) { struct clk_pllv4 *pll; diff --git a/drivers/clk/imx/clk-sccg-pll.c b/drivers/clk/imx/clk-sscg-pll.c index 5d65f65c512e..acd1b9002be6 100644 --- a/drivers/clk/imx/clk-sccg-pll.c +++ b/drivers/clk/imx/clk-sscg-pll.c @@ -67,7 +67,7 @@ #define PLL_SCCG_LOCK_TIMEOUT 70 -struct clk_sccg_pll_setup { +struct clk_sscg_pll_setup { int divr1, divf1; int divr2, divf2; int divq; @@ -83,22 +83,22 @@ struct clk_sccg_pll_setup { int fout_error; }; -struct clk_sccg_pll { +struct clk_sscg_pll { struct clk_hw hw; const struct clk_ops ops; void __iomem *base; - struct clk_sccg_pll_setup setup; + struct clk_sscg_pll_setup setup; u8 parent; u8 bypass1; u8 bypass2; }; -#define to_clk_sccg_pll(_hw) container_of(_hw, struct clk_sccg_pll, hw) +#define to_clk_sscg_pll(_hw) container_of(_hw, struct clk_sscg_pll, hw) -static int clk_sccg_pll_wait_lock(struct clk_sccg_pll *pll) +static int clk_sscg_pll_wait_lock(struct clk_sscg_pll *pll) { u32 val; @@ -112,15 +112,15 @@ static int clk_sccg_pll_wait_lock(struct clk_sccg_pll *pll) return 0; } -static int clk_sccg_pll2_check_match(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup) +static int clk_sscg_pll2_check_match(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup) { int new_diff = temp_setup->fout - temp_setup->fout_request; int diff = temp_setup->fout_error; if (abs(diff) > abs(new_diff)) { temp_setup->fout_error = new_diff; - memcpy(setup, temp_setup, sizeof(struct clk_sccg_pll_setup)); + memcpy(setup, temp_setup, sizeof(struct clk_sscg_pll_setup)); if (temp_setup->fout_request == temp_setup->fout) return 0; @@ -128,8 +128,8 @@ static int clk_sccg_pll2_check_match(struct clk_sccg_pll_setup *setup, return -1; } -static int clk_sccg_divq_lookup(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup) +static int clk_sscg_divq_lookup(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup) { int ret = -EINVAL; @@ -144,7 +144,7 @@ static int clk_sccg_divq_lookup(struct clk_sccg_pll_setup *setup, temp_setup->fout = temp_setup->vco2; do_div(temp_setup->fout, 2 * (temp_setup->divq + 1)); - ret = clk_sccg_pll2_check_match(setup, temp_setup); + ret = clk_sscg_pll2_check_match(setup, temp_setup); if (!ret) { temp_setup->bypass = PLL_BYPASS1; return ret; @@ -155,14 +155,14 @@ static int clk_sccg_divq_lookup(struct clk_sccg_pll_setup *setup, return ret; } -static int clk_sccg_divf2_lookup(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup) +static int clk_sscg_divf2_lookup(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup) { int ret = -EINVAL; for (temp_setup->divf2 = 0; temp_setup->divf2 <= PLL_DIVF2_MAX; temp_setup->divf2++) { - ret = clk_sccg_divq_lookup(setup, temp_setup); + ret = clk_sscg_divq_lookup(setup, temp_setup); if (!ret) return ret; } @@ -170,8 +170,8 @@ static int clk_sccg_divf2_lookup(struct clk_sccg_pll_setup *setup, return ret; } -static int clk_sccg_divr2_lookup(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup) +static int clk_sscg_divr2_lookup(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup) { int ret = -EINVAL; @@ -181,7 +181,7 @@ static int clk_sccg_divr2_lookup(struct clk_sccg_pll_setup *setup, do_div(temp_setup->ref_div2, temp_setup->divr2 + 1); if (temp_setup->ref_div2 >= PLL_STAGE2_REF_MIN_FREQ && temp_setup->ref_div2 <= PLL_STAGE2_REF_MAX_FREQ) { - ret = clk_sccg_divf2_lookup(setup, temp_setup); + ret = clk_sscg_divf2_lookup(setup, temp_setup); if (!ret) return ret; } @@ -190,8 +190,8 @@ static int clk_sccg_divr2_lookup(struct clk_sccg_pll_setup *setup, return ret; } -static int clk_sccg_pll2_find_setup(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup, +static int clk_sscg_pll2_find_setup(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup, uint64_t ref) { @@ -202,12 +202,12 @@ static int clk_sccg_pll2_find_setup(struct clk_sccg_pll_setup *setup, temp_setup->vco1 = ref; - ret = clk_sccg_divr2_lookup(setup, temp_setup); + ret = clk_sscg_divr2_lookup(setup, temp_setup); return ret; } -static int clk_sccg_divf1_lookup(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup) +static int clk_sscg_divf1_lookup(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup) { int ret = -EINVAL; @@ -219,7 +219,7 @@ static int clk_sccg_divf1_lookup(struct clk_sccg_pll_setup *setup, vco1 *= 2; vco1 *= temp_setup->divf1 + 1; - ret = clk_sccg_pll2_find_setup(setup, temp_setup, vco1); + ret = clk_sscg_pll2_find_setup(setup, temp_setup, vco1); if (!ret) { temp_setup->bypass = PLL_BYPASS_NONE; return ret; @@ -229,8 +229,8 @@ static int clk_sccg_divf1_lookup(struct clk_sccg_pll_setup *setup, return ret; } -static int clk_sccg_divr1_lookup(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup) +static int clk_sscg_divr1_lookup(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup) { int ret = -EINVAL; @@ -240,7 +240,7 @@ static int clk_sccg_divr1_lookup(struct clk_sccg_pll_setup *setup, do_div(temp_setup->ref_div1, temp_setup->divr1 + 1); if (temp_setup->ref_div1 >= PLL_STAGE1_REF_MIN_FREQ && temp_setup->ref_div1 <= PLL_STAGE1_REF_MAX_FREQ) { - ret = clk_sccg_divf1_lookup(setup, temp_setup); + ret = clk_sscg_divf1_lookup(setup, temp_setup); if (!ret) return ret; } @@ -249,8 +249,8 @@ static int clk_sccg_divr1_lookup(struct clk_sccg_pll_setup *setup, return ret; } -static int clk_sccg_pll1_find_setup(struct clk_sccg_pll_setup *setup, - struct clk_sccg_pll_setup *temp_setup, +static int clk_sscg_pll1_find_setup(struct clk_sscg_pll_setup *setup, + struct clk_sscg_pll_setup *temp_setup, uint64_t ref) { @@ -261,20 +261,20 @@ static int clk_sccg_pll1_find_setup(struct clk_sccg_pll_setup *setup, temp_setup->ref = ref; - ret = clk_sccg_divr1_lookup(setup, temp_setup); + ret = clk_sscg_divr1_lookup(setup, temp_setup); return ret; } -static int clk_sccg_pll_find_setup(struct clk_sccg_pll_setup *setup, +static int clk_sscg_pll_find_setup(struct clk_sscg_pll_setup *setup, uint64_t prate, uint64_t rate, int try_bypass) { - struct clk_sccg_pll_setup temp_setup; + struct clk_sscg_pll_setup temp_setup; int ret = -EINVAL; - memset(&temp_setup, 0, sizeof(struct clk_sccg_pll_setup)); - memset(setup, 0, sizeof(struct clk_sccg_pll_setup)); + memset(&temp_setup, 0, sizeof(struct clk_sscg_pll_setup)); + memset(setup, 0, sizeof(struct clk_sscg_pll_setup)); temp_setup.fout_error = PLL_OUT_MAX_FREQ; temp_setup.fout_request = rate; @@ -290,11 +290,11 @@ static int clk_sccg_pll_find_setup(struct clk_sccg_pll_setup *setup, break; case PLL_BYPASS1: - ret = clk_sccg_pll2_find_setup(setup, &temp_setup, prate); + ret = clk_sscg_pll2_find_setup(setup, &temp_setup, prate); break; case PLL_BYPASS_NONE: - ret = clk_sccg_pll1_find_setup(setup, &temp_setup, prate); + ret = clk_sscg_pll1_find_setup(setup, &temp_setup, prate); break; } @@ -302,30 +302,30 @@ static int clk_sccg_pll_find_setup(struct clk_sccg_pll_setup *setup, } -static int clk_sccg_pll_is_prepared(struct clk_hw *hw) +static int clk_sscg_pll_is_prepared(struct clk_hw *hw) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); u32 val = readl_relaxed(pll->base + PLL_CFG0); return (val & PLL_PD_MASK) ? 0 : 1; } -static int clk_sccg_pll_prepare(struct clk_hw *hw) +static int clk_sscg_pll_prepare(struct clk_hw *hw) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); u32 val; val = readl_relaxed(pll->base + PLL_CFG0); val &= ~PLL_PD_MASK; writel_relaxed(val, pll->base + PLL_CFG0); - return clk_sccg_pll_wait_lock(pll); + return clk_sscg_pll_wait_lock(pll); } -static void clk_sccg_pll_unprepare(struct clk_hw *hw) +static void clk_sscg_pll_unprepare(struct clk_hw *hw) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); u32 val; val = readl_relaxed(pll->base + PLL_CFG0); @@ -333,10 +333,10 @@ static void clk_sccg_pll_unprepare(struct clk_hw *hw) writel_relaxed(val, pll->base + PLL_CFG0); } -static unsigned long clk_sccg_pll_recalc_rate(struct clk_hw *hw, +static unsigned long clk_sscg_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); u32 val, divr1, divf1, divr2, divf2, divq; u64 temp64; @@ -364,11 +364,11 @@ static unsigned long clk_sccg_pll_recalc_rate(struct clk_hw *hw, return temp64; } -static int clk_sccg_pll_set_rate(struct clk_hw *hw, unsigned long rate, +static int clk_sscg_pll_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); - struct clk_sccg_pll_setup *setup = &pll->setup; + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); + struct clk_sscg_pll_setup *setup = &pll->setup; u32 val; /* set bypass here too since the parent might be the same */ @@ -387,12 +387,12 @@ static int clk_sccg_pll_set_rate(struct clk_hw *hw, unsigned long rate, val |= FIELD_PREP(PLL_DIVQ_MASK, setup->divq); writel_relaxed(val, pll->base + PLL_CFG2); - return clk_sccg_pll_wait_lock(pll); + return clk_sscg_pll_wait_lock(pll); } -static u8 clk_sccg_pll_get_parent(struct clk_hw *hw) +static u8 clk_sscg_pll_get_parent(struct clk_hw *hw) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); u32 val; u8 ret = pll->parent; @@ -404,9 +404,9 @@ static u8 clk_sccg_pll_get_parent(struct clk_hw *hw) return ret; } -static int clk_sccg_pll_set_parent(struct clk_hw *hw, u8 index) +static int clk_sscg_pll_set_parent(struct clk_hw *hw, u8 index) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); u32 val; val = readl(pll->base + PLL_CFG0); @@ -414,18 +414,18 @@ static int clk_sccg_pll_set_parent(struct clk_hw *hw, u8 index) val |= FIELD_PREP(SSCG_PLL_BYPASS_MASK, pll->setup.bypass); writel(val, pll->base + PLL_CFG0); - return clk_sccg_pll_wait_lock(pll); + return clk_sscg_pll_wait_lock(pll); } -static int __clk_sccg_pll_determine_rate(struct clk_hw *hw, +static int __clk_sscg_pll_determine_rate(struct clk_hw *hw, struct clk_rate_request *req, uint64_t min, uint64_t max, uint64_t rate, int bypass) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); - struct clk_sccg_pll_setup *setup = &pll->setup; + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); + struct clk_sscg_pll_setup *setup = &pll->setup; struct clk_hw *parent_hw = NULL; int bypass_parent_index; int ret = -EINVAL; @@ -448,7 +448,7 @@ static int __clk_sccg_pll_determine_rate(struct clk_hw *hw, parent_hw = clk_hw_get_parent_by_index(hw, bypass_parent_index); ret = __clk_determine_rate(parent_hw, req); if (!ret) { - ret = clk_sccg_pll_find_setup(setup, req->rate, + ret = clk_sscg_pll_find_setup(setup, req->rate, rate, bypass); } @@ -459,11 +459,11 @@ static int __clk_sccg_pll_determine_rate(struct clk_hw *hw, return ret; } -static int clk_sccg_pll_determine_rate(struct clk_hw *hw, +static int clk_sscg_pll_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { - struct clk_sccg_pll *pll = to_clk_sccg_pll(hw); - struct clk_sccg_pll_setup *setup = &pll->setup; + struct clk_sscg_pll *pll = to_clk_sscg_pll(hw); + struct clk_sscg_pll_setup *setup = &pll->setup; uint64_t rate = req->rate; uint64_t min = req->min_rate; uint64_t max = req->max_rate; @@ -472,18 +472,18 @@ static int clk_sccg_pll_determine_rate(struct clk_hw *hw, if (rate < PLL_OUT_MIN_FREQ || rate > PLL_OUT_MAX_FREQ) return ret; - ret = __clk_sccg_pll_determine_rate(hw, req, req->rate, req->rate, + ret = __clk_sscg_pll_determine_rate(hw, req, req->rate, req->rate, rate, PLL_BYPASS2); if (!ret) return ret; - ret = __clk_sccg_pll_determine_rate(hw, req, PLL_STAGE1_REF_MIN_FREQ, + ret = __clk_sscg_pll_determine_rate(hw, req, PLL_STAGE1_REF_MIN_FREQ, PLL_STAGE1_REF_MAX_FREQ, rate, PLL_BYPASS1); if (!ret) return ret; - ret = __clk_sccg_pll_determine_rate(hw, req, PLL_REF_MIN_FREQ, + ret = __clk_sscg_pll_determine_rate(hw, req, PLL_REF_MIN_FREQ, PLL_REF_MAX_FREQ, rate, PLL_BYPASS_NONE); if (!ret) @@ -495,25 +495,25 @@ static int clk_sccg_pll_determine_rate(struct clk_hw *hw, return ret; } -static const struct clk_ops clk_sccg_pll_ops = { - .prepare = clk_sccg_pll_prepare, - .unprepare = clk_sccg_pll_unprepare, - .is_prepared = clk_sccg_pll_is_prepared, - .recalc_rate = clk_sccg_pll_recalc_rate, - .set_rate = clk_sccg_pll_set_rate, - .set_parent = clk_sccg_pll_set_parent, - .get_parent = clk_sccg_pll_get_parent, - .determine_rate = clk_sccg_pll_determine_rate, +static const struct clk_ops clk_sscg_pll_ops = { + .prepare = clk_sscg_pll_prepare, + .unprepare = clk_sscg_pll_unprepare, + .is_prepared = clk_sscg_pll_is_prepared, + .recalc_rate = clk_sscg_pll_recalc_rate, + .set_rate = clk_sscg_pll_set_rate, + .set_parent = clk_sscg_pll_set_parent, + .get_parent = clk_sscg_pll_get_parent, + .determine_rate = clk_sscg_pll_determine_rate, }; -struct clk *imx_clk_sccg_pll(const char *name, +struct clk_hw *imx_clk_hw_sscg_pll(const char *name, const char * const *parent_names, u8 num_parents, u8 parent, u8 bypass1, u8 bypass2, void __iomem *base, unsigned long flags) { - struct clk_sccg_pll *pll; + struct clk_sscg_pll *pll; struct clk_init_data init; struct clk_hw *hw; int ret; @@ -528,7 +528,7 @@ struct clk *imx_clk_sccg_pll(const char *name, pll->base = base; init.name = name; - init.ops = &clk_sccg_pll_ops; + init.ops = &clk_sscg_pll_ops; init.flags = flags; init.parent_names = parent_names; @@ -545,5 +545,5 @@ struct clk *imx_clk_sccg_pll(const char *name, return ERR_PTR(ret); } - return hw->clk; + return hw; } diff --git a/drivers/clk/imx/clk.c b/drivers/clk/imx/clk.c index cfc05e43c343..87ab8db3d282 100644 --- a/drivers/clk/imx/clk.c +++ b/drivers/clk/imx/clk.c @@ -22,6 +22,14 @@ void imx_unregister_clocks(struct clk *clks[], unsigned int count) clk_unregister(clks[i]); } +void imx_unregister_hw_clocks(struct clk_hw *hws[], unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) + clk_hw_unregister(hws[i]); +} + void __init imx_mmdc_mask_handshake(void __iomem *ccm_base, unsigned int chn) { @@ -94,8 +102,8 @@ struct clk_hw * __init imx_obtain_fixed_clock_hw( return __clk_get_hw(clk); } -struct clk_hw * __init imx_obtain_fixed_clk_hw(struct device_node *np, - const char *name) +struct clk_hw * imx_obtain_fixed_clk_hw(struct device_node *np, + const char *name) { struct clk *clk; diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h index bc5bb6ac8636..b05213b91dcf 100644 --- a/drivers/clk/imx/clk.h +++ b/drivers/clk/imx/clk.h @@ -12,6 +12,7 @@ void imx_check_clk_hws(struct clk_hw *clks[], unsigned int count); void imx_register_uart_clocks(struct clk ** const clks[]); void imx_mmdc_mask_handshake(void __iomem *ccm_base, unsigned int chn); void imx_unregister_clocks(struct clk *clks[], unsigned int count); +void imx_unregister_hw_clocks(struct clk_hw *hws[], unsigned int count); extern void imx_cscmr1_fixup(u32 *val); @@ -24,7 +25,7 @@ enum imx_pllv1_type { IMX_PLLV1_IMX35, }; -enum imx_sccg_pll_type { +enum imx_sscg_pll_type { SCCG_PLL1, SCCG_PLL2, }; @@ -52,64 +53,98 @@ struct imx_pll14xx_clk { extern struct imx_pll14xx_clk imx_1416x_pll; extern struct imx_pll14xx_clk imx_1443x_pll; +extern struct imx_pll14xx_clk imx_1443x_dram_pll; #define imx_clk_cpu(name, parent_name, div, mux, pll, step) \ - imx_clk_hw_cpu(name, parent_name, div, mux, pll, step)->clk + to_clk(imx_clk_hw_cpu(name, parent_name, div, mux, pll, step)) #define clk_register_gate2(dev, name, parent_name, flags, reg, bit_idx, \ cgr_val, clk_gate_flags, lock, share_count) \ - clk_hw_register_gate2(dev, name, parent_name, flags, reg, bit_idx, \ - cgr_val, clk_gate_flags, lock, share_count)->clk + to_clk(clk_hw_register_gate2(dev, name, parent_name, flags, reg, bit_idx, \ + cgr_val, clk_gate_flags, lock, share_count)) #define imx_clk_pllv3(type, name, parent_name, base, div_mask) \ - imx_clk_hw_pllv3(type, name, parent_name, base, div_mask)->clk + to_clk(imx_clk_hw_pllv3(type, name, parent_name, base, div_mask)) #define imx_clk_pfd(name, parent_name, reg, idx) \ - imx_clk_hw_pfd(name, parent_name, reg, idx)->clk + to_clk(imx_clk_hw_pfd(name, parent_name, reg, idx)) #define imx_clk_gate_exclusive(name, parent, reg, shift, exclusive_mask) \ - imx_clk_hw_gate_exclusive(name, parent, reg, shift, exclusive_mask)->clk + to_clk(imx_clk_hw_gate_exclusive(name, parent, reg, shift, exclusive_mask)) + +#define imx_clk_fixed(name, rate) \ + to_clk(imx_clk_hw_fixed(name, rate)) #define imx_clk_fixed_factor(name, parent, mult, div) \ - imx_clk_hw_fixed_factor(name, parent, mult, div)->clk + to_clk(imx_clk_hw_fixed_factor(name, parent, mult, div)) + +#define imx_clk_divider(name, parent, reg, shift, width) \ + to_clk(imx_clk_hw_divider(name, parent, reg, shift, width)) #define imx_clk_divider2(name, parent, reg, shift, width) \ - imx_clk_hw_divider2(name, parent, reg, shift, width)->clk + to_clk(imx_clk_hw_divider2(name, parent, reg, shift, width)) + +#define imx_clk_divider_flags(name, parent, reg, shift, width, flags) \ + to_clk(imx_clk_hw_divider_flags(name, parent, reg, shift, width, flags)) + +#define imx_clk_gate(name, parent, reg, shift) \ + to_clk(imx_clk_hw_gate(name, parent, reg, shift)) #define imx_clk_gate_dis(name, parent, reg, shift) \ - imx_clk_hw_gate_dis(name, parent, reg, shift)->clk + to_clk(imx_clk_hw_gate_dis(name, parent, reg, shift)) #define imx_clk_gate2(name, parent, reg, shift) \ - imx_clk_hw_gate2(name, parent, reg, shift)->clk + to_clk(imx_clk_hw_gate2(name, parent, reg, shift)) #define imx_clk_gate2_flags(name, parent, reg, shift, flags) \ - imx_clk_hw_gate2_flags(name, parent, reg, shift, flags)->clk + to_clk(imx_clk_hw_gate2_flags(name, parent, reg, shift, flags)) #define imx_clk_gate2_shared2(name, parent, reg, shift, share_count) \ - imx_clk_hw_gate2_shared2(name, parent, reg, shift, share_count)->clk + to_clk(imx_clk_hw_gate2_shared2(name, parent, reg, shift, share_count)) #define imx_clk_gate3(name, parent, reg, shift) \ - imx_clk_hw_gate3(name, parent, reg, shift)->clk + to_clk(imx_clk_hw_gate3(name, parent, reg, shift)) #define imx_clk_gate4(name, parent, reg, shift) \ - imx_clk_hw_gate4(name, parent, reg, shift)->clk + to_clk(imx_clk_hw_gate4(name, parent, reg, shift)) #define imx_clk_mux(name, reg, shift, width, parents, num_parents) \ - imx_clk_hw_mux(name, reg, shift, width, parents, num_parents)->clk + to_clk(imx_clk_hw_mux(name, reg, shift, width, parents, num_parents)) + +#define imx_clk_pllv1(type, name, parent, base) \ + to_clk(imx_clk_hw_pllv1(type, name, parent, base)) + +#define imx_clk_pllv2(name, parent, base) \ + to_clk(imx_clk_hw_pllv2(name, parent, base)) + +#define imx_clk_frac_pll(name, parent_name, base) \ + to_clk(imx_clk_hw_frac_pll(name, parent_name, base)) + +#define imx_clk_sscg_pll(name, parent_names, num_parents, parent,\ + bypass1, bypass2, base, flags) \ + to_clk(imx_clk_hw_sscg_pll(name, parent_names, num_parents, parent,\ + bypass1, bypass2, base, flags)) struct clk *imx_clk_pll14xx(const char *name, const char *parent_name, void __iomem *base, const struct imx_pll14xx_clk *pll_clk); -struct clk *imx_clk_pllv1(enum imx_pllv1_type type, const char *name, +#define imx_clk_pll14xx(name, parent_name, base, pll_clk) \ + to_clk(imx_clk_hw_pll14xx(name, parent_name, base, pll_clk)) + +struct clk_hw *imx_clk_hw_pll14xx(const char *name, const char *parent_name, + void __iomem *base, + const struct imx_pll14xx_clk *pll_clk); + +struct clk_hw *imx_clk_hw_pllv1(enum imx_pllv1_type type, const char *name, const char *parent, void __iomem *base); -struct clk *imx_clk_pllv2(const char *name, const char *parent, +struct clk_hw *imx_clk_hw_pllv2(const char *name, const char *parent, void __iomem *base); -struct clk *imx_clk_frac_pll(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_frac_pll(const char *name, const char *parent_name, void __iomem *base); -struct clk *imx_clk_sccg_pll(const char *name, +struct clk_hw *imx_clk_hw_sscg_pll(const char *name, const char * const *parent_names, u8 num_parents, u8 parent, u8 bypass1, u8 bypass2, @@ -149,7 +184,7 @@ struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name, .kdiv = (_k), \ } -struct clk_hw *imx_clk_pllv4(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_pllv4(const char *name, const char *parent_name, void __iomem *base); struct clk_hw *clk_hw_register_gate2(struct device *dev, const char *name, @@ -173,7 +208,7 @@ struct clk_hw *imx_clk_hw_gate_exclusive(const char *name, const char *parent, struct clk_hw *imx_clk_hw_pfd(const char *name, const char *parent_name, void __iomem *reg, u8 idx); -struct clk_hw *imx_clk_pfdv2(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_pfdv2(const char *name, const char *parent_name, void __iomem *reg, u8 idx); struct clk_hw *imx_clk_hw_busy_divider(const char *name, const char *parent_name, @@ -184,7 +219,7 @@ struct clk_hw *imx_clk_hw_busy_mux(const char *name, void __iomem *reg, u8 shift u8 width, void __iomem *busy_reg, u8 busy_shift, const char * const *parent_names, int num_parents); -struct clk_hw *imx7ulp_clk_composite(const char *name, +struct clk_hw *imx7ulp_clk_hw_composite(const char *name, const char * const *parent_names, int num_parents, bool mux_present, bool rate_present, bool gate_present, @@ -198,9 +233,11 @@ struct clk_hw *imx_clk_hw_fixup_mux(const char *name, void __iomem *reg, u8 shift, u8 width, const char * const *parents, int num_parents, void (*fixup)(u32 *val)); -static inline struct clk *imx_clk_fixed(const char *name, int rate) +static inline struct clk *to_clk(struct clk_hw *hw) { - return clk_register_fixed_rate(NULL, name, NULL, 0, rate); + if (IS_ERR_OR_NULL(hw)) + return ERR_CAST(hw); + return hw->clk; } static inline struct clk_hw *imx_clk_hw_fixed(const char *name, int rate) @@ -224,13 +261,6 @@ static inline struct clk_hw *imx_clk_hw_fixed_factor(const char *name, CLK_SET_RATE_PARENT, mult, div); } -static inline struct clk *imx_clk_divider(const char *name, const char *parent, - void __iomem *reg, u8 shift, u8 width) -{ - return clk_register_divider(NULL, name, parent, CLK_SET_RATE_PARENT, - reg, shift, width, 0, &imx_ccm_lock); -} - static inline struct clk_hw *imx_clk_hw_divider(const char *name, const char *parent, void __iomem *reg, u8 shift, @@ -240,14 +270,6 @@ static inline struct clk_hw *imx_clk_hw_divider(const char *name, reg, shift, width, 0, &imx_ccm_lock); } -static inline struct clk *imx_clk_divider_flags(const char *name, - const char *parent, void __iomem *reg, u8 shift, u8 width, - unsigned long flags) -{ - return clk_register_divider(NULL, name, parent, flags, - reg, shift, width, 0, &imx_ccm_lock); -} - static inline struct clk_hw *imx_clk_hw_divider_flags(const char *name, const char *parent, void __iomem *reg, u8 shift, @@ -274,13 +296,6 @@ static inline struct clk *imx_clk_divider2_flags(const char *name, reg, shift, width, 0, &imx_ccm_lock); } -static inline struct clk *imx_clk_gate(const char *name, const char *parent, - void __iomem *reg, u8 shift) -{ - return clk_register_gate(NULL, name, parent, CLK_SET_RATE_PARENT, reg, - shift, 0, &imx_ccm_lock); -} - static inline struct clk_hw *imx_clk_hw_gate_flags(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned long flags) { @@ -355,15 +370,18 @@ static inline struct clk_hw *imx_clk_hw_gate3(const char *name, const char *pare reg, shift, 0, &imx_ccm_lock); } -static inline struct clk *imx_clk_gate3_flags(const char *name, +static inline struct clk_hw *imx_clk_hw_gate3_flags(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned long flags) { - return clk_register_gate(NULL, name, parent, + return clk_hw_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, reg, shift, 0, &imx_ccm_lock); } +#define imx_clk_gate3_flags(name, parent, reg, shift, flags) \ + to_clk(imx_clk_hw_gate3_flags(name, parent, reg, shift, flags)) + static inline struct clk_hw *imx_clk_hw_gate4(const char *name, const char *parent, void __iomem *reg, u8 shift) { @@ -372,15 +390,18 @@ static inline struct clk_hw *imx_clk_hw_gate4(const char *name, const char *pare reg, shift, 0x3, 0, &imx_ccm_lock, NULL); } -static inline struct clk *imx_clk_gate4_flags(const char *name, +static inline struct clk_hw *imx_clk_hw_gate4_flags(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned long flags) { - return clk_register_gate2(NULL, name, parent, + return clk_hw_register_gate2(NULL, name, parent, flags | CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, reg, shift, 0x3, 0, &imx_ccm_lock, NULL); } +#define imx_clk_gate4_flags(name, parent, reg, shift, flags) \ + to_clk(imx_clk_hw_gate4_flags(name, parent, reg, shift, flags)) + static inline struct clk_hw *imx_clk_hw_mux(const char *name, void __iomem *reg, u8 shift, u8 width, const char * const *parents, int num_parents) @@ -420,6 +441,16 @@ static inline struct clk *imx_clk_mux_flags(const char *name, &imx_ccm_lock); } +static inline struct clk_hw *imx_clk_hw_mux2_flags(const char *name, + void __iomem *reg, u8 shift, u8 width, + const char * const *parents, + int num_parents, unsigned long flags) +{ + return clk_hw_register_mux(NULL, name, parents, num_parents, + flags | CLK_SET_RATE_NO_REPARENT | CLK_OPS_PARENT_ENABLE, + reg, shift, width, 0, &imx_ccm_lock); +} + static inline struct clk *imx_clk_mux2_flags(const char *name, void __iomem *reg, u8 shift, u8 width, const char * const *parents, @@ -446,23 +477,38 @@ struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name, struct clk *div, struct clk *mux, struct clk *pll, struct clk *step); -struct clk *imx8m_clk_composite_flags(const char *name, - const char * const *parent_names, - int num_parents, void __iomem *reg, - unsigned long flags); +struct clk_hw *imx8m_clk_hw_composite_flags(const char *name, + const char * const *parent_names, + int num_parents, + void __iomem *reg, + unsigned long flags); -#define __imx8m_clk_composite(name, parent_names, reg, flags) \ - imx8m_clk_composite_flags(name, parent_names, \ +#define imx8m_clk_composite_flags(name, parent_names, num_parents, reg, \ + flags) \ + to_clk(imx8m_clk_hw_composite_flags(name, parent_names, \ + num_parents, reg, flags)) + +#define __imx8m_clk_hw_composite(name, parent_names, reg, flags) \ + imx8m_clk_hw_composite_flags(name, parent_names, \ ARRAY_SIZE(parent_names), reg, \ flags | CLK_SET_RATE_NO_REPARENT | CLK_OPS_PARENT_ENABLE) +#define __imx8m_clk_composite(name, parent_names, reg, flags) \ + to_clk(__imx8m_clk_hw_composite(name, parent_names, reg, flags)) + +#define imx8m_clk_hw_composite(name, parent_names, reg) \ + __imx8m_clk_hw_composite(name, parent_names, reg, 0) + #define imx8m_clk_composite(name, parent_names, reg) \ __imx8m_clk_composite(name, parent_names, reg, 0) +#define imx8m_clk_hw_composite_critical(name, parent_names, reg) \ + __imx8m_clk_hw_composite(name, parent_names, reg, CLK_IS_CRITICAL) + #define imx8m_clk_composite_critical(name, parent_names, reg) \ __imx8m_clk_composite(name, parent_names, reg, CLK_IS_CRITICAL) -struct clk_hw *imx_clk_divider_gate(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_divider_gate(const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 shift, u8 width, u8 clk_divider_flags, const struct clk_div_table *table, spinlock_t *lock); diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index 7efc3617bbd5..ea3c70d1307e 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -174,36 +174,36 @@ config COMMON_CLK_MT6779_AUDSYS This driver supports Mediatek MT6779 audsys clocks. config COMMON_CLK_MT6797 - bool "Clock driver for MediaTek MT6797" - depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST - select COMMON_CLK_MEDIATEK - default ARCH_MEDIATEK && ARM64 - ---help--- - This driver supports MediaTek MT6797 basic clocks. + bool "Clock driver for MediaTek MT6797" + depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK && ARM64 + ---help--- + This driver supports MediaTek MT6797 basic clocks. config COMMON_CLK_MT6797_MMSYS - bool "Clock driver for MediaTek MT6797 mmsys" - depends on COMMON_CLK_MT6797 - ---help--- - This driver supports MediaTek MT6797 mmsys clocks. + bool "Clock driver for MediaTek MT6797 mmsys" + depends on COMMON_CLK_MT6797 + ---help--- + This driver supports MediaTek MT6797 mmsys clocks. config COMMON_CLK_MT6797_IMGSYS - bool "Clock driver for MediaTek MT6797 imgsys" - depends on COMMON_CLK_MT6797 - ---help--- - This driver supports MediaTek MT6797 imgsys clocks. + bool "Clock driver for MediaTek MT6797 imgsys" + depends on COMMON_CLK_MT6797 + ---help--- + This driver supports MediaTek MT6797 imgsys clocks. config COMMON_CLK_MT6797_VDECSYS - bool "Clock driver for MediaTek MT6797 vdecsys" - depends on COMMON_CLK_MT6797 - ---help--- - This driver supports MediaTek MT6797 vdecsys clocks. + bool "Clock driver for MediaTek MT6797 vdecsys" + depends on COMMON_CLK_MT6797 + ---help--- + This driver supports MediaTek MT6797 vdecsys clocks. config COMMON_CLK_MT6797_VENCSYS - bool "Clock driver for MediaTek MT6797 vencsys" - depends on COMMON_CLK_MT6797 - ---help--- - This driver supports MediaTek MT6797 vencsys clocks. + bool "Clock driver for MediaTek MT6797 vencsys" + depends on COMMON_CLK_MT6797 + ---help--- + This driver supports MediaTek MT6797 vencsys clocks. config COMMON_CLK_MT7622 bool "Clock driver for MediaTek MT7622" diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile index 3939f218587a..6eca2a406ee3 100644 --- a/drivers/clk/meson/Makefile +++ b/drivers/clk/meson/Makefile @@ -18,4 +18,4 @@ obj-$(CONFIG_COMMON_CLK_AXG) += axg.o axg-aoclk.o obj-$(CONFIG_COMMON_CLK_AXG_AUDIO) += axg-audio.o obj-$(CONFIG_COMMON_CLK_GXBB) += gxbb.o gxbb-aoclk.o obj-$(CONFIG_COMMON_CLK_G12A) += g12a.o g12a-aoclk.o -obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b.o +obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b.o meson8-ddr.o diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c index 2d39a8bc367c..fc9df4860872 100644 --- a/drivers/clk/meson/clk-mpll.c +++ b/drivers/clk/meson/clk-mpll.c @@ -129,7 +129,7 @@ static int mpll_set_rate(struct clk_hw *hw, return 0; } -static void mpll_init(struct clk_hw *hw) +static int mpll_init(struct clk_hw *hw) { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_mpll_data *mpll = meson_clk_mpll_data(clk); @@ -151,6 +151,8 @@ static void mpll_init(struct clk_hw *hw) /* Set the magic misc bit if required */ if (MESON_PARM_APPLICABLE(&mpll->misc)) meson_parm_write(clk->map, &mpll->misc, 1); + + return 0; } const struct clk_ops meson_clk_mpll_ro_ops = { diff --git a/drivers/clk/meson/clk-phase.c b/drivers/clk/meson/clk-phase.c index 80c3ada193a4..fe22e171121a 100644 --- a/drivers/clk/meson/clk-phase.c +++ b/drivers/clk/meson/clk-phase.c @@ -78,7 +78,7 @@ meson_clk_triphase_data(struct clk_regmap *clk) return (struct meson_clk_triphase_data *)clk->data; } -static void meson_clk_triphase_sync(struct clk_hw *hw) +static int meson_clk_triphase_sync(struct clk_hw *hw) { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_triphase_data *tph = meson_clk_triphase_data(clk); @@ -88,6 +88,8 @@ static void meson_clk_triphase_sync(struct clk_hw *hw) val = meson_parm_read(clk->map, &tph->ph0); meson_parm_write(clk->map, &tph->ph1, val); meson_parm_write(clk->map, &tph->ph2, val); + + return 0; } static int meson_clk_triphase_get_phase(struct clk_hw *hw) diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index ddb1e5634739..b17a13e9337c 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -77,6 +77,15 @@ static unsigned long meson_clk_pll_recalc_rate(struct clk_hw *hw, unsigned int m, n, frac; n = meson_parm_read(clk->map, &pll->n); + + /* + * On some HW, N is set to zero on init. This value is invalid as + * it would result in a division by zero. The rate can't be + * calculated in this case + */ + if (n == 0) + return 0; + m = meson_parm_read(clk->map, &pll->m); frac = MESON_PARM_APPLICABLE(&pll->frac) ? @@ -277,7 +286,7 @@ static int meson_clk_pll_wait_lock(struct clk_hw *hw) return -ETIMEDOUT; } -static void meson_clk_pll_init(struct clk_hw *hw) +static int meson_clk_pll_init(struct clk_hw *hw) { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_pll_data *pll = meson_clk_pll_data(clk); @@ -288,6 +297,8 @@ static void meson_clk_pll_init(struct clk_hw *hw) pll->init_count); meson_parm_write(clk->map, &pll->rst, 0); } + + return 0; } static int meson_clk_pll_is_enabled(struct clk_hw *hw) diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index b3af61cc6fb9..d2760a021301 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -4692,6 +4692,7 @@ static struct clk_regmap *const g12a_clk_regmaps[] = { &g12a_bt656, &g12a_usb1_to_ddr, &g12a_mmc_pclk, + &g12a_uart2, &g12a_vpu_intr, &g12a_gic, &g12a_sd_emmc_a_clk0, diff --git a/drivers/clk/meson/meson8-ddr.c b/drivers/clk/meson/meson8-ddr.c new file mode 100644 index 000000000000..4b73ea244b63 --- /dev/null +++ b/drivers/clk/meson/meson8-ddr.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Amlogic Meson8 DDR clock controller + * + * Copyright (C) 2019 Martin Blumenstingl <martin.blumenstingl@googlemail.com> + */ + +#include <dt-bindings/clock/meson8-ddr-clkc.h> + +#include <linux/clk-provider.h> +#include <linux/platform_device.h> + +#include "clk-regmap.h" +#include "clk-pll.h" + +#define AM_DDR_PLL_CNTL 0x00 +#define AM_DDR_PLL_CNTL1 0x04 +#define AM_DDR_PLL_CNTL2 0x08 +#define AM_DDR_PLL_CNTL3 0x0c +#define AM_DDR_PLL_CNTL4 0x10 +#define AM_DDR_PLL_STS 0x14 +#define DDR_CLK_CNTL 0x18 +#define DDR_CLK_STS 0x1c + +static struct clk_regmap meson8_ddr_pll_dco = { + .data = &(struct meson_clk_pll_data){ + .en = { + .reg_off = AM_DDR_PLL_CNTL, + .shift = 30, + .width = 1, + }, + .m = { + .reg_off = AM_DDR_PLL_CNTL, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = AM_DDR_PLL_CNTL, + .shift = 9, + .width = 5, + }, + .l = { + .reg_off = AM_DDR_PLL_CNTL, + .shift = 31, + .width = 1, + }, + .rst = { + .reg_off = AM_DDR_PLL_CNTL, + .shift = 29, + .width = 1, + }, + }, + .hw.init = &(struct clk_init_data){ + .name = "ddr_pll_dco", + .ops = &meson_clk_pll_ro_ops, + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xtal", + }, + .num_parents = 1, + }, +}; + +static struct clk_regmap meson8_ddr_pll = { + .data = &(struct clk_regmap_div_data){ + .offset = AM_DDR_PLL_CNTL, + .shift = 16, + .width = 2, + .flags = CLK_DIVIDER_POWER_OF_TWO, + }, + .hw.init = &(struct clk_init_data){ + .name = "ddr_pll", + .ops = &clk_regmap_divider_ro_ops, + .parent_hws = (const struct clk_hw *[]) { + &meson8_ddr_pll_dco.hw + }, + .num_parents = 1, + }, +}; + +static struct clk_hw_onecell_data meson8_ddr_clk_hw_onecell_data = { + .hws = { + [DDR_CLKID_DDR_PLL_DCO] = &meson8_ddr_pll_dco.hw, + [DDR_CLKID_DDR_PLL] = &meson8_ddr_pll.hw, + }, + .num = 2, +}; + +static struct clk_regmap *const meson8_ddr_clk_regmaps[] = { + &meson8_ddr_pll_dco, + &meson8_ddr_pll, +}; + +static const struct regmap_config meson8_ddr_clkc_regmap_config = { + .reg_bits = 8, + .val_bits = 32, + .reg_stride = 4, + .max_register = DDR_CLK_STS, +}; + +static int meson8_ddr_clkc_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + void __iomem *base; + struct clk_hw *hw; + int ret, i; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + regmap = devm_regmap_init_mmio(&pdev->dev, base, + &meson8_ddr_clkc_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* Populate regmap */ + for (i = 0; i < ARRAY_SIZE(meson8_ddr_clk_regmaps); i++) + meson8_ddr_clk_regmaps[i]->map = regmap; + + /* Register all clks */ + for (i = 0; i < meson8_ddr_clk_hw_onecell_data.num; i++) { + hw = meson8_ddr_clk_hw_onecell_data.hws[i]; + + ret = devm_clk_hw_register(&pdev->dev, hw); + if (ret) { + dev_err(&pdev->dev, "Clock registration failed\n"); + return ret; + } + } + + return devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, + &meson8_ddr_clk_hw_onecell_data); +} + +static const struct of_device_id meson8_ddr_clkc_match_table[] = { + { .compatible = "amlogic,meson8-ddr-clkc" }, + { .compatible = "amlogic,meson8b-ddr-clkc" }, + { /* sentinel */ } +}; + +static struct platform_driver meson8_ddr_clkc_driver = { + .probe = meson8_ddr_clkc_probe, + .driver = { + .name = "meson8-ddr-clkc", + .of_match_table = meson8_ddr_clkc_match_table, + }, +}; + +builtin_platform_driver(meson8_ddr_clkc_driver); diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c index 67e6691e080c..9fd31f23b2a9 100644 --- a/drivers/clk/meson/meson8b.c +++ b/drivers/clk/meson/meson8b.c @@ -97,8 +97,10 @@ static struct clk_regmap meson8b_fixed_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "fixed_pll_dco", .ops = &meson_clk_pll_ro_ops, - .parent_hws = (const struct clk_hw *[]) { - &meson8b_xtal.hw + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xtal", + .name = "xtal", + .index = -1, }, .num_parents = 1, }, @@ -162,8 +164,10 @@ static struct clk_regmap meson8b_hdmi_pll_dco = { /* sometimes also called "HPLL" or "HPLL PLL" */ .name = "hdmi_pll_dco", .ops = &meson_clk_pll_ro_ops, - .parent_hws = (const struct clk_hw *[]) { - &meson8b_xtal.hw + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xtal", + .name = "xtal", + .index = -1, }, .num_parents = 1, }, @@ -237,8 +241,10 @@ static struct clk_regmap meson8b_sys_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "sys_pll_dco", .ops = &meson_clk_pll_ops, - .parent_hws = (const struct clk_hw *[]) { - &meson8b_xtal.hw + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xtal", + .name = "xtal", + .index = -1, }, .num_parents = 1, }, @@ -631,9 +637,9 @@ static struct clk_regmap meson8b_cpu_in_sel = { .hw.init = &(struct clk_init_data){ .name = "cpu_in_sel", .ops = &clk_regmap_mux_ops, - .parent_hws = (const struct clk_hw *[]) { - &meson8b_xtal.hw, - &meson8b_sys_pll.hw, + .parent_data = (const struct clk_parent_data[]) { + { .fw_name = "xtal", .name = "xtal", .index = -1, }, + { .hw = &meson8b_sys_pll.hw, }, }, .num_parents = 2, .flags = (CLK_SET_RATE_PARENT | @@ -736,9 +742,9 @@ static struct clk_regmap meson8b_cpu_clk = { .hw.init = &(struct clk_init_data){ .name = "cpu_clk", .ops = &clk_regmap_mux_ops, - .parent_hws = (const struct clk_hw *[]) { - &meson8b_xtal.hw, - &meson8b_cpu_scale_out_sel.hw, + .parent_data = (const struct clk_parent_data[]) { + { .fw_name = "xtal", .name = "xtal", .index = -1, }, + { .hw = &meson8b_cpu_scale_out_sel.hw, }, }, .num_parents = 2, .flags = (CLK_SET_RATE_PARENT | @@ -758,12 +764,12 @@ static struct clk_regmap meson8b_nand_clk_sel = { .name = "nand_clk_sel", .ops = &clk_regmap_mux_ops, /* FIXME all other parents are unknown: */ - .parent_hws = (const struct clk_hw *[]) { - &meson8b_fclk_div4.hw, - &meson8b_fclk_div3.hw, - &meson8b_fclk_div5.hw, - &meson8b_fclk_div7.hw, - &meson8b_xtal.hw, + .parent_data = (const struct clk_parent_data[]) { + { .hw = &meson8b_fclk_div4.hw, }, + { .hw = &meson8b_fclk_div3.hw, }, + { .hw = &meson8b_fclk_div5.hw, }, + { .hw = &meson8b_fclk_div7.hw, }, + { .fw_name = "xtal", .name = "xtal", .index = -1, }, }, .num_parents = 5, .flags = CLK_SET_RATE_PARENT, @@ -1721,8 +1727,10 @@ static struct clk_regmap meson8b_hdmi_sys_sel = { .name = "hdmi_sys_sel", .ops = &clk_regmap_mux_ro_ops, /* FIXME: all other parents are unknown */ - .parent_hws = (const struct clk_hw *[]) { - &meson8b_xtal.hw + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xtal", + .name = "xtal", + .index = -1, }, .num_parents = 1, .flags = CLK_SET_RATE_NO_REPARENT, @@ -1764,17 +1772,20 @@ static struct clk_regmap meson8b_hdmi_sys = { /* * The MALI IP is clocked by two identical clocks (mali_0 and mali_1) - * muxed by a glitch-free switch on Meson8b and Meson8m2. Meson8 only - * has mali_0 and no glitch-free mux. + * muxed by a glitch-free switch on Meson8b and Meson8m2. The CCF can + * actually manage this glitch-free mux because it does top-to-bottom + * updates the each clock tree and switches to the "inactive" one when + * CLK_SET_RATE_GATE is set. + * Meson8 only has mali_0 and no glitch-free mux. */ -static const struct clk_hw *meson8b_mali_0_1_parent_hws[] = { - &meson8b_xtal.hw, - &meson8b_mpll2.hw, - &meson8b_mpll1.hw, - &meson8b_fclk_div7.hw, - &meson8b_fclk_div4.hw, - &meson8b_fclk_div3.hw, - &meson8b_fclk_div5.hw, +static const struct clk_parent_data meson8b_mali_0_1_parent_data[] = { + { .fw_name = "xtal", .name = "xtal", .index = -1, }, + { .hw = &meson8b_mpll2.hw, }, + { .hw = &meson8b_mpll1.hw, }, + { .hw = &meson8b_fclk_div7.hw, }, + { .hw = &meson8b_fclk_div4.hw, }, + { .hw = &meson8b_fclk_div3.hw, }, + { .hw = &meson8b_fclk_div5.hw, }, }; static u32 meson8b_mali_0_1_mux_table[] = { 0, 2, 3, 4, 5, 6, 7 }; @@ -1789,8 +1800,8 @@ static struct clk_regmap meson8b_mali_0_sel = { .hw.init = &(struct clk_init_data){ .name = "mali_0_sel", .ops = &clk_regmap_mux_ops, - .parent_hws = meson8b_mali_0_1_parent_hws, - .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_hws), + .parent_data = meson8b_mali_0_1_parent_data, + .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_data), /* * Don't propagate rate changes up because the only changeable * parents are mpll1 and mpll2 but we need those for audio and @@ -1830,7 +1841,7 @@ static struct clk_regmap meson8b_mali_0 = { &meson8b_mali_0_div.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT, }, }; @@ -1844,8 +1855,8 @@ static struct clk_regmap meson8b_mali_1_sel = { .hw.init = &(struct clk_init_data){ .name = "mali_1_sel", .ops = &clk_regmap_mux_ops, - .parent_hws = meson8b_mali_0_1_parent_hws, - .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_hws), + .parent_data = meson8b_mali_0_1_parent_data, + .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_data), /* * Don't propagate rate changes up because the only changeable * parents are mpll1 and mpll2 but we need those for audio and @@ -1885,7 +1896,7 @@ static struct clk_regmap meson8b_mali_1 = { &meson8b_mali_1_div.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT, }, }; @@ -1944,8 +1955,10 @@ static struct clk_regmap meson8m2_gp_pll_dco = { .hw.init = &(struct clk_init_data){ .name = "gp_pll_dco", .ops = &meson_clk_pll_ops, - .parent_hws = (const struct clk_hw *[]) { - &meson8b_xtal.hw + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xtal", + .name = "xtal", + .index = -1, }, .num_parents = 1, }, @@ -3585,7 +3598,7 @@ static const struct reset_control_ops meson8b_clk_reset_ops = { struct meson8b_nb_data { struct notifier_block nb; - struct clk_hw_onecell_data *onecell_data; + struct clk_hw *cpu_clk; }; static int meson8b_cpu_clk_notifier_cb(struct notifier_block *nb, @@ -3593,30 +3606,25 @@ static int meson8b_cpu_clk_notifier_cb(struct notifier_block *nb, { struct meson8b_nb_data *nb_data = container_of(nb, struct meson8b_nb_data, nb); - struct clk_hw **hws = nb_data->onecell_data->hws; - struct clk_hw *cpu_clk_hw, *parent_clk_hw; - struct clk *cpu_clk, *parent_clk; + struct clk_hw *parent_clk; int ret; switch (event) { case PRE_RATE_CHANGE: - parent_clk_hw = hws[CLKID_XTAL]; + /* xtal */ + parent_clk = clk_hw_get_parent_by_index(nb_data->cpu_clk, 0); break; case POST_RATE_CHANGE: - parent_clk_hw = hws[CLKID_CPU_SCALE_OUT_SEL]; + /* cpu_scale_out_sel */ + parent_clk = clk_hw_get_parent_by_index(nb_data->cpu_clk, 1); break; default: return NOTIFY_DONE; } - cpu_clk_hw = hws[CLKID_CPUCLK]; - cpu_clk = __clk_lookup(clk_hw_get_name(cpu_clk_hw)); - - parent_clk = __clk_lookup(clk_hw_get_name(parent_clk_hw)); - - ret = clk_set_parent(cpu_clk, parent_clk); + ret = clk_hw_set_parent(nb_data->cpu_clk, parent_clk); if (ret) return notifier_from_errno(ret); @@ -3682,20 +3690,26 @@ static void __init meson8b_clkc_init_common(struct device_node *np, meson8b_clk_regmaps[i]->map = map; /* - * register all clks - * CLKID_UNUSED = 0, so skip it and start with CLKID_XTAL = 1 + * always skip CLKID_UNUSED and also skip XTAL if the .dtb provides the + * XTAL clock as input. */ - for (i = CLKID_XTAL; i < CLK_NR_CLKS; i++) { + if (!IS_ERR(of_clk_get_by_name(np, "xtal"))) + i = CLKID_PLL_FIXED; + else + i = CLKID_XTAL; + + /* register all clks */ + for (; i < CLK_NR_CLKS; i++) { /* array might be sparse */ if (!clk_hw_onecell_data->hws[i]) continue; - ret = clk_hw_register(NULL, clk_hw_onecell_data->hws[i]); + ret = of_clk_hw_register(np, clk_hw_onecell_data->hws[i]); if (ret) return; } - meson8b_cpu_nb_data.onecell_data = clk_hw_onecell_data; + meson8b_cpu_nb_data.cpu_clk = clk_hw_onecell_data->hws[CLKID_CPUCLK]; /* * FIXME we shouldn't program the muxes in notifier handlers. The diff --git a/drivers/clk/meson/sclk-div.c b/drivers/clk/meson/sclk-div.c index 3acf03780221..76d31c0a3342 100644 --- a/drivers/clk/meson/sclk-div.c +++ b/drivers/clk/meson/sclk-div.c @@ -216,7 +216,7 @@ static int sclk_div_is_enabled(struct clk_hw *hw) return 0; } -static void sclk_div_init(struct clk_hw *hw) +static int sclk_div_init(struct clk_hw *hw) { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_sclk_div_data *sclk = meson_sclk_div_data(clk); @@ -231,6 +231,8 @@ static void sclk_div_init(struct clk_hw *hw) sclk->cached_div = val + 1; sclk_div_get_duty_cycle(hw, &sclk->cached_duty); + + return 0; } const struct clk_ops meson_sclk_div_ops = { diff --git a/drivers/clk/microchip/clk-core.c b/drivers/clk/microchip/clk-core.c index 567755d6f844..1b4f023cdc8b 100644 --- a/drivers/clk/microchip/clk-core.c +++ b/drivers/clk/microchip/clk-core.c @@ -266,10 +266,12 @@ static void roclk_disable(struct clk_hw *hw) writel(REFO_ON | REFO_OE, PIC32_CLR(refo->ctrl_reg)); } -static void roclk_init(struct clk_hw *hw) +static int roclk_init(struct clk_hw *hw) { /* initialize clock in disabled state */ roclk_disable(hw); + + return 0; } static u8 roclk_get_parent(struct clk_hw *hw) @@ -880,7 +882,7 @@ static int sclk_set_parent(struct clk_hw *hw, u8 index) return err; } -static void sclk_init(struct clk_hw *hw) +static int sclk_init(struct clk_hw *hw) { struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw); unsigned long flags; @@ -899,6 +901,8 @@ static void sclk_init(struct clk_hw *hw) writel(v, sclk->slew_reg); spin_unlock_irqrestore(&sclk->core->reg_lock, flags); } + + return 0; } /* sclk with post-divider */ diff --git a/drivers/clk/mmp/clk-frac.c b/drivers/clk/mmp/clk-frac.c index 90bf181f191a..fabc09aca6c4 100644 --- a/drivers/clk/mmp/clk-frac.c +++ b/drivers/clk/mmp/clk-frac.c @@ -109,7 +109,7 @@ static int clk_factor_set_rate(struct clk_hw *hw, unsigned long drate, return 0; } -static void clk_factor_init(struct clk_hw *hw) +static int clk_factor_init(struct clk_hw *hw) { struct mmp_clk_factor *factor = to_clk_factor(hw); struct mmp_clk_factor_masks *masks = factor->masks; @@ -146,6 +146,8 @@ static void clk_factor_init(struct clk_hw *hw) if (factor->lock) spin_unlock_irqrestore(factor->lock, flags); + + return 0; } static const struct clk_ops clk_factor_ops = { diff --git a/drivers/clk/mmp/clk-mix.c b/drivers/clk/mmp/clk-mix.c index 90814b2613c0..d2cd36c54474 100644 --- a/drivers/clk/mmp/clk-mix.c +++ b/drivers/clk/mmp/clk-mix.c @@ -419,12 +419,14 @@ static int mmp_clk_set_rate(struct clk_hw *hw, unsigned long rate, } } -static void mmp_clk_mix_init(struct clk_hw *hw) +static int mmp_clk_mix_init(struct clk_hw *hw) { struct mmp_clk_mix *mix = to_clk_mix(hw); if (mix->table) _filter_clk_table(mix, mix->table, mix->table_size); + + return 0; } const struct clk_ops mmp_clk_mix_ops = { diff --git a/drivers/clk/mvebu/Kconfig b/drivers/clk/mvebu/Kconfig index 415e6906a113..ded07b0bd0d5 100644 --- a/drivers/clk/mvebu/Kconfig +++ b/drivers/clk/mvebu/Kconfig @@ -29,7 +29,7 @@ config ARMADA_39X_CLK select MVEBU_CLK_COMMON config ARMADA_37XX_CLK - bool + bool config ARMADA_XP_CLK bool diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 3b33ef129274..15cdcdc9b3b8 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config KRAIT_CLOCKS - bool - select KRAIT_L2_ACCESSORS + bool + select KRAIT_L2_ACCESSORS config QCOM_GDSC bool @@ -14,6 +14,7 @@ menuconfig COMMON_CLK_QCOM tristate "Support for Qualcomm's clock controllers" depends on OF depends on ARCH_QCOM || COMPILE_TEST + select RATIONAL select REGMAP_MMIO select RESET_CONTROLLER @@ -95,6 +96,14 @@ config IPQ_GCC_4019 Say Y if you want to use peripheral devices such as UART, SPI, i2c, USB, SD/eMMC, etc. +config IPQ_GCC_6018 + tristate "IPQ6018 Global Clock Controller" + help + Support for global clock controller on ipq6018 devices. + Say Y if you want to use peripheral devices such as UART, SPI, + i2c, USB, SD/eMMC, etc. Select this for the root clock + of ipq6018. + config IPQ_GCC_806X tristate "IPQ806x Global Clock Controller" help @@ -229,6 +238,15 @@ config MSM_GPUCC_8998 Say Y if you want to support graphics controller devices and functionality such as 3D graphics. +config MSM_MMCC_8998 + tristate "MSM8998 Multimedia Clock Controller" + select MSM_GCC_8998 + select QCOM_GDSC + help + Support for the multimedia clock controller on msm8998 devices. + Say Y if you want to support multimedia devices such as display, + graphics, video encode/decode, camera, etc. + config QCS_GCC_404 tristate "QCS404 Global Clock Controller" help @@ -236,6 +254,15 @@ config QCS_GCC_404 Say Y if you want to use multimedia devices or peripheral devices such as UART, SPI, I2C, USB, SD/eMMC, PCIe etc. +config SC_DISPCC_7180 + tristate "SC7180 Display Clock Controller" + select SC_GCC_7180 + help + Support for the display clock controller on Qualcomm Technologies, Inc + SC7180 devices. + Say Y if you want to support display devices and functionality such as + splash screen. + config SC_GCC_7180 tristate "SC7180 Global Clock Controller" select QCOM_GDSC @@ -245,6 +272,22 @@ config SC_GCC_7180 Say Y if you want to use peripheral devices such as UART, SPI, I2C, USB, UFS, SDCC, etc. +config SC_GPUCC_7180 + tristate "SC7180 Graphics Clock Controller" + select SC_GCC_7180 + help + Support for the graphics clock controller on SC7180 devices. + Say Y if you want to support graphics controller devices and + functionality such as 3D graphics. + +config SC_VIDEOCC_7180 + tristate "SC7180 Video Clock Controller" + select SC_GCC_7180 + help + Support for the video clock controller on SC7180 devices. + Say Y if you want to support video devices and functionality such as + video encode and decode. + config SDM_CAMCC_845 tristate "SDM845 Camera Clock Controller" select SDM_GCC_845 diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index d899661d0f44..656a87e629d4 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -20,6 +20,7 @@ clk-qcom-$(CONFIG_QCOM_GDSC) += gdsc.o obj-$(CONFIG_APQ_GCC_8084) += gcc-apq8084.o obj-$(CONFIG_APQ_MMCC_8084) += mmcc-apq8084.o obj-$(CONFIG_IPQ_GCC_4019) += gcc-ipq4019.o +obj-$(CONFIG_IPQ_GCC_6018) += gcc-ipq6018.o obj-$(CONFIG_IPQ_GCC_806X) += gcc-ipq806x.o obj-$(CONFIG_IPQ_GCC_8074) += gcc-ipq8074.o obj-$(CONFIG_IPQ_LCC_806X) += lcc-ipq806x.o @@ -37,6 +38,7 @@ obj-$(CONFIG_MSM_GPUCC_8998) += gpucc-msm8998.o obj-$(CONFIG_MSM_MMCC_8960) += mmcc-msm8960.o obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o +obj-$(CONFIG_MSM_MMCC_8998) += mmcc-msm8998.o obj-$(CONFIG_QCOM_A53PLL) += a53-pll.o obj-$(CONFIG_QCOM_CLK_APCS_MSM8916) += apcs-msm8916.o obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o @@ -45,7 +47,10 @@ obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o obj-$(CONFIG_QCS_GCC_404) += gcc-qcs404.o obj-$(CONFIG_QCS_Q6SSTOP_404) += q6sstop-qcs404.o obj-$(CONFIG_QCS_TURING_404) += turingcc-qcs404.o +obj-$(CONFIG_SC_DISPCC_7180) += dispcc-sc7180.o obj-$(CONFIG_SC_GCC_7180) += gcc-sc7180.o +obj-$(CONFIG_SC_GPUCC_7180) += gpucc-sc7180.o +obj-$(CONFIG_SC_VIDEOCC_7180) += videocc-sc7180.o obj-$(CONFIG_SDM_CAMCC_845) += camcc-sdm845.o obj-$(CONFIG_SDM_DISPCC_845) += dispcc-sdm845.o obj-$(CONFIG_SDM_GCC_660) += gcc-sdm660.o diff --git a/drivers/clk/qcom/apcs-msm8916.c b/drivers/clk/qcom/apcs-msm8916.c index a6c89a310b18..cf69a97d0439 100644 --- a/drivers/clk/qcom/apcs-msm8916.c +++ b/drivers/clk/qcom/apcs-msm8916.c @@ -19,9 +19,9 @@ static const u32 gpll0_a53cc_map[] = { 4, 5 }; -static const char * const gpll0_a53cc[] = { - "gpll0_vote", - "a53pll", +static const struct clk_parent_data pdata[] = { + { .fw_name = "aux", .name = "gpll0_vote", }, + { .fw_name = "pll", .name = "a53pll", }, }; /* @@ -62,8 +62,8 @@ static int qcom_apcs_msm8916_clk_probe(struct platform_device *pdev) return -ENOMEM; init.name = "a53mux"; - init.parent_names = gpll0_a53cc; - init.num_parents = ARRAY_SIZE(gpll0_a53cc); + init.parent_data = pdata; + init.num_parents = ARRAY_SIZE(pdata); init.ops = &clk_regmap_mux_div_ops; init.flags = CLK_SET_RATE_PARENT; @@ -79,7 +79,8 @@ static int qcom_apcs_msm8916_clk_probe(struct platform_device *pdev) a53cc->pclk = devm_clk_get(parent, NULL); if (IS_ERR(a53cc->pclk)) { ret = PTR_ERR(a53cc->pclk); - dev_err(dev, "failed to get clk: %d\n", ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "failed to get clk: %d\n", ret); return ret; } diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 055318f97991..7c2936da9b14 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -878,6 +878,14 @@ static long clk_trion_pll_round_rate(struct clk_hw *hw, unsigned long rate, return clamp(rate, min_freq, max_freq); } +const struct clk_ops clk_alpha_pll_fixed_ops = { + .enable = clk_alpha_pll_enable, + .disable = clk_alpha_pll_disable, + .is_enabled = clk_alpha_pll_is_enabled, + .recalc_rate = clk_alpha_pll_recalc_rate, +}; +EXPORT_SYMBOL_GPL(clk_alpha_pll_fixed_ops); + const struct clk_ops clk_alpha_pll_ops = { .enable = clk_alpha_pll_enable, .disable = clk_alpha_pll_disable, @@ -1024,6 +1032,25 @@ void clk_fabia_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, regmap_write(regmap, PLL_CONFIG_CTL(pll), config->config_ctl_val); + if (config->config_ctl_hi_val) + regmap_write(regmap, PLL_CONFIG_CTL_U(pll), + config->config_ctl_hi_val); + + if (config->user_ctl_val) + regmap_write(regmap, PLL_USER_CTL(pll), config->user_ctl_val); + + if (config->user_ctl_hi_val) + regmap_write(regmap, PLL_USER_CTL_U(pll), + config->user_ctl_hi_val); + + if (config->test_ctl_val) + regmap_write(regmap, PLL_TEST_CTL(pll), + config->test_ctl_val); + + if (config->test_ctl_hi_val) + regmap_write(regmap, PLL_TEST_CTL_U(pll), + config->test_ctl_hi_val); + if (config->post_div_mask) { mask = config->post_div_mask; val = config->post_div_val; @@ -1141,14 +1168,9 @@ static int alpha_pll_fabia_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long prate) { struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); - u32 val, l, alpha_width = pll_alpha_width(pll); + u32 l, alpha_width = pll_alpha_width(pll); u64 a; unsigned long rrate; - int ret = 0; - - ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val); - if (ret) - return ret; rrate = alpha_pll_round_rate(rate, prate, &l, &a, alpha_width); @@ -1167,7 +1189,64 @@ static int alpha_pll_fabia_set_rate(struct clk_hw *hw, unsigned long rate, return __clk_alpha_pll_update_latch(pll); } +static int alpha_pll_fabia_prepare(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + const struct pll_vco *vco; + struct clk_hw *parent_hw; + unsigned long cal_freq, rrate; + u32 cal_l, val, alpha_width = pll_alpha_width(pll); + u64 a; + int ret; + + /* Check if calibration needs to be done i.e. PLL is in reset */ + ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val); + if (ret) + return ret; + + /* Return early if calibration is not needed. */ + if (val & PLL_RESET_N) + return 0; + + vco = alpha_pll_find_vco(pll, clk_hw_get_rate(hw)); + if (!vco) { + pr_err("alpha pll: not in a valid vco range\n"); + return -EINVAL; + } + + cal_freq = DIV_ROUND_CLOSEST((pll->vco_table[0].min_freq + + pll->vco_table[0].max_freq) * 54, 100); + + parent_hw = clk_hw_get_parent(hw); + if (!parent_hw) + return -EINVAL; + + rrate = alpha_pll_round_rate(cal_freq, clk_hw_get_rate(parent_hw), + &cal_l, &a, alpha_width); + /* + * Due to a limited number of bits for fractional rate programming, the + * rounded up rate could be marginally higher than the requested rate. + */ + if (rrate > (cal_freq + FABIA_PLL_RATE_MARGIN) || rrate < cal_freq) + return -EINVAL; + + /* Setup PLL for calibration frequency */ + regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL(pll), cal_l); + + /* Bringup the PLL at calibration frequency */ + ret = clk_alpha_pll_enable(hw); + if (ret) { + pr_err("alpha pll calibration failed\n"); + return ret; + } + + clk_alpha_pll_disable(hw); + + return 0; +} + const struct clk_ops clk_alpha_pll_fabia_ops = { + .prepare = alpha_pll_fabia_prepare, .enable = alpha_pll_fabia_enable, .disable = alpha_pll_fabia_disable, .is_enabled = clk_alpha_pll_is_enabled, diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index 15f27f4b06df..fbc1f67c7a26 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -94,6 +94,10 @@ struct alpha_pll_config { u32 alpha_hi; u32 config_ctl_val; u32 config_ctl_hi_val; + u32 user_ctl_val; + u32 user_ctl_hi_val; + u32 test_ctl_val; + u32 test_ctl_hi_val; u32 main_output_mask; u32 aux_output_mask; u32 aux2_output_mask; @@ -109,6 +113,7 @@ struct alpha_pll_config { }; extern const struct clk_ops clk_alpha_pll_ops; +extern const struct clk_ops clk_alpha_pll_fixed_ops; extern const struct clk_ops clk_alpha_pll_hwfsm_ops; extern const struct clk_ops clk_alpha_pll_postdiv_ops; extern const struct clk_ops clk_alpha_pll_huayra_ops; diff --git a/drivers/clk/qcom/clk-hfpll.c b/drivers/clk/qcom/clk-hfpll.c index 3c04805f2a55..e847d586a73a 100644 --- a/drivers/clk/qcom/clk-hfpll.c +++ b/drivers/clk/qcom/clk-hfpll.c @@ -196,7 +196,7 @@ static unsigned long clk_hfpll_recalc_rate(struct clk_hw *hw, return l_val * parent_rate; } -static void clk_hfpll_init(struct clk_hw *hw) +static int clk_hfpll_init(struct clk_hw *hw) { struct clk_hfpll *h = to_clk_hfpll(hw); struct hfpll_data const *hd = h->d; @@ -206,7 +206,7 @@ static void clk_hfpll_init(struct clk_hw *hw) regmap_read(regmap, hd->mode_reg, &mode); if (mode != (PLL_BYPASSNL | PLL_RESET_N | PLL_OUTCTRL)) { __clk_hfpll_init_once(hw); - return; + return 0; } if (hd->status_reg) { @@ -218,6 +218,8 @@ static void clk_hfpll_init(struct clk_hw *hw) __clk_hfpll_init_once(hw); } } + + return 0; } static int hfpll_is_enabled(struct clk_hw *hw) diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index 78358b81d249..86d2b8b90173 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -161,6 +161,7 @@ extern const struct clk_ops clk_byte2_ops; extern const struct clk_ops clk_pixel_ops; extern const struct clk_ops clk_gfx3d_ops; extern const struct clk_ops clk_rcg2_shared_ops; +extern const struct clk_ops clk_dp_ops; struct clk_rcg_dfs_data { struct clk_rcg2 *rcg; diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 8f4b9bec2956..357159fe85b5 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -10,6 +10,7 @@ #include <linux/export.h> #include <linux/clk-provider.h> #include <linux/delay.h> +#include <linux/rational.h> #include <linux/regmap.h> #include <linux/math64.h> #include <linux/slab.h> @@ -217,6 +218,9 @@ static int _freq_tbl_determine_rate(struct clk_hw *hw, const struct freq_tbl *f, clk_flags = clk_hw_get_flags(hw); p = clk_hw_get_parent_by_index(hw, index); + if (!p) + return -EINVAL; + if (clk_flags & CLK_SET_RATE_PARENT) { rate = f->freq; if (f->pre_div) { @@ -952,7 +956,7 @@ static void clk_rcg2_dfs_populate_freq(struct clk_hw *hw, unsigned int l, struct clk_rcg2 *rcg = to_clk_rcg2(hw); struct clk_hw *p; unsigned long prate = 0; - u32 val, mask, cfg, mode; + u32 val, mask, cfg, mode, src; int i, num_parents; regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + SE_PERF_DFSR(l), &cfg); @@ -962,12 +966,12 @@ static void clk_rcg2_dfs_populate_freq(struct clk_hw *hw, unsigned int l, if (cfg & mask) f->pre_div = cfg & mask; - cfg &= CFG_SRC_SEL_MASK; - cfg >>= CFG_SRC_SEL_SHIFT; + src = cfg & CFG_SRC_SEL_MASK; + src >>= CFG_SRC_SEL_SHIFT; num_parents = clk_hw_get_num_parents(hw); for (i = 0; i < num_parents; i++) { - if (cfg == rcg->parent_map[i].cfg) { + if (src == rcg->parent_map[i].cfg) { f->src = rcg->parent_map[i].src; p = clk_hw_get_parent_by_index(&rcg->clkr.hw, i); prate = clk_hw_get_rate(p); @@ -1124,3 +1128,79 @@ int qcom_cc_register_rcg_dfs(struct regmap *regmap, return 0; } EXPORT_SYMBOL_GPL(qcom_cc_register_rcg_dfs); + +static int clk_rcg2_dp_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + struct freq_tbl f = { 0 }; + u32 mask = BIT(rcg->hid_width) - 1; + u32 hid_div, cfg; + int i, num_parents = clk_hw_get_num_parents(hw); + unsigned long num, den; + + rational_best_approximation(parent_rate, rate, + GENMASK(rcg->mnd_width - 1, 0), + GENMASK(rcg->mnd_width - 1, 0), &den, &num); + + if (!num || !den) + return -EINVAL; + + regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, &cfg); + hid_div = cfg; + cfg &= CFG_SRC_SEL_MASK; + cfg >>= CFG_SRC_SEL_SHIFT; + + for (i = 0; i < num_parents; i++) { + if (cfg == rcg->parent_map[i].cfg) { + f.src = rcg->parent_map[i].src; + break; + } + } + + f.pre_div = hid_div; + f.pre_div >>= CFG_SRC_DIV_SHIFT; + f.pre_div &= mask; + + if (num != den) { + f.m = num; + f.n = den; + } else { + f.m = 0; + f.n = 0; + } + + return clk_rcg2_configure(rcg, &f); +} + +static int clk_rcg2_dp_set_rate_and_parent(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate, u8 index) +{ + return clk_rcg2_dp_set_rate(hw, rate, parent_rate); +} + +static int clk_rcg2_dp_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_rate_request parent_req = *req; + int ret; + + ret = __clk_determine_rate(clk_hw_get_parent(hw), &parent_req); + if (ret) + return ret; + + req->best_parent_rate = parent_req.rate; + + return 0; +} + +const struct clk_ops clk_dp_ops = { + .is_enabled = clk_rcg2_is_enabled, + .get_parent = clk_rcg2_get_parent, + .set_parent = clk_rcg2_set_parent, + .recalc_rate = clk_rcg2_recalc_rate, + .set_rate = clk_rcg2_dp_set_rate, + .set_rate_and_parent = clk_rcg2_dp_set_rate_and_parent, + .determine_rate = clk_rcg2_dp_determine_rate, +}; +EXPORT_SYMBOL_GPL(clk_dp_ops); diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index 7ed313ad6e43..98a118c1e244 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -396,6 +396,7 @@ static struct clk_hw *sc7180_rpmh_clocks[] = { [RPMH_RF_CLK1_A] = &sdm845_rf_clk1_ao.hw, [RPMH_RF_CLK2] = &sdm845_rf_clk2.hw, [RPMH_RF_CLK2_A] = &sdm845_rf_clk2_ao.hw, + [RPMH_IPA_CLK] = &sdm845_ipa.hw, }; static const struct clk_rpmh_desc clk_rpmh_sc7180 = { @@ -431,11 +432,16 @@ static int clk_rpmh_probe(struct platform_device *pdev) hw_clks = desc->clks; for (i = 0; i < desc->num_clks; i++) { - const char *name = hw_clks[i]->init->name; + const char *name; u32 res_addr; size_t aux_data_len; const struct bcm_db *data; + if (!hw_clks[i]) + continue; + + name = hw_clks[i]->init->name; + rpmh_clk = to_clk_rpmh(hw_clks[i]); res_addr = cmd_db_read_addr(rpmh_clk->res_name); if (!res_addr) { @@ -481,9 +487,9 @@ static int clk_rpmh_probe(struct platform_device *pdev) } static const struct of_device_id clk_rpmh_match_table[] = { + { .compatible = "qcom,sc7180-rpmh-clk", .data = &clk_rpmh_sc7180}, { .compatible = "qcom,sdm845-rpmh-clk", .data = &clk_rpmh_sdm845}, { .compatible = "qcom,sm8150-rpmh-clk", .data = &clk_rpmh_sm8150}, - { .compatible = "qcom,sc7180-rpmh-clk", .data = &clk_rpmh_sc7180}, { } }; MODULE_DEVICE_TABLE(of, clk_rpmh_match_table); diff --git a/drivers/clk/qcom/clk-smd-rpm.c b/drivers/clk/qcom/clk-smd-rpm.c index 930fa4a4c52a..0bbfef9fa6de 100644 --- a/drivers/clk/qcom/clk-smd-rpm.c +++ b/drivers/clk/qcom/clk-smd-rpm.c @@ -485,6 +485,8 @@ static struct clk_smd_rpm *msm8974_clks[] = { [RPM_SMD_MMSSNOC_AHB_CLK] = &msm8974_mmssnoc_ahb_clk, [RPM_SMD_MMSSNOC_AHB_A_CLK] = &msm8974_mmssnoc_ahb_a_clk, [RPM_SMD_BIMC_CLK] = &msm8974_bimc_clk, + [RPM_SMD_GFX3D_CLK_SRC] = &msm8974_gfx3d_clk_src, + [RPM_SMD_GFX3D_A_CLK_SRC] = &msm8974_gfx3d_a_clk_src, [RPM_SMD_BIMC_A_CLK] = &msm8974_bimc_a_clk, [RPM_SMD_OCMEMGX_CLK] = &msm8974_ocmemgx_clk, [RPM_SMD_OCMEMGX_A_CLK] = &msm8974_ocmemgx_a_clk, @@ -648,6 +650,7 @@ static const struct rpm_smd_clk_desc rpm_clk_qcs404 = { }; /* msm8998 */ +DEFINE_CLK_SMD_RPM(msm8998, bimc_clk, bimc_a_clk, QCOM_SMD_RPM_MEM_CLK, 0); DEFINE_CLK_SMD_RPM(msm8998, pcnoc_clk, pcnoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 0); DEFINE_CLK_SMD_RPM(msm8998, snoc_clk, snoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 1); DEFINE_CLK_SMD_RPM(msm8998, cnoc_clk, cnoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 2); @@ -671,6 +674,8 @@ DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8998, rf_clk2_pin, rf_clk2_a_pin, 5); DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8998, rf_clk3, rf_clk3_a, 6); DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8998, rf_clk3_pin, rf_clk3_a_pin, 6); static struct clk_smd_rpm *msm8998_clks[] = { + [RPM_SMD_BIMC_CLK] = &msm8998_bimc_clk, + [RPM_SMD_BIMC_A_CLK] = &msm8998_bimc_a_clk, [RPM_SMD_PCNOC_CLK] = &msm8998_pcnoc_clk, [RPM_SMD_PCNOC_A_CLK] = &msm8998_pcnoc_a_clk, [RPM_SMD_SNOC_CLK] = &msm8998_snoc_clk, diff --git a/drivers/clk/qcom/dispcc-sc7180.c b/drivers/clk/qcom/dispcc-sc7180.c new file mode 100644 index 000000000000..dd7af41e47eb --- /dev/null +++ b/drivers/clk/qcom/dispcc-sc7180.c @@ -0,0 +1,761 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include <dt-bindings/clock/qcom,dispcc-sc7180.h> + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap-divider.h" +#include "common.h" +#include "gdsc.h" + +enum { + P_BI_TCXO, + P_CHIP_SLEEP_CLK, + P_CORE_BI_PLL_TEST_SE, + P_DISP_CC_PLL0_OUT_EVEN, + P_DISP_CC_PLL0_OUT_MAIN, + P_DP_PHY_PLL_LINK_CLK, + P_DP_PHY_PLL_VCO_DIV_CLK, + P_DSI0_PHY_PLL_OUT_BYTECLK, + P_DSI0_PHY_PLL_OUT_DSICLK, + P_GPLL0_OUT_MAIN, +}; + +static const struct pll_vco fabia_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct clk_alpha_pll disp_cc_pll0 = { + .offset = 0x0, + .vco_table = fabia_vco, + .num_vco = ARRAY_SIZE(fabia_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_pll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fabia_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_disp_cc_pll0_out_even[] = { + { 0x0, 1 }, + { } +}; + +static struct clk_alpha_pll_postdiv disp_cc_pll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_disp_cc_pll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_disp_cc_pll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_pll0_out_even", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_pll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static const struct parent_map disp_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map disp_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_DP_PHY_PLL_LINK_CLK, 1 }, + { P_DP_PHY_PLL_VCO_DIV_CLK, 2 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "dp_phy_pll_link_clk" }, + { .fw_name = "dp_phy_pll_vco_div_clk" }, +}; + +static const struct parent_map disp_cc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_BYTECLK, 1 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "dsi0_phy_pll_out_byteclk" }, +}; + +static const struct parent_map disp_cc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_DISP_CC_PLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_MAIN, 4 }, + { P_DISP_CC_PLL0_OUT_EVEN, 5 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &disp_cc_pll0.clkr.hw }, + { .fw_name = "gcc_disp_gpll0_clk_src" }, + { .hw = &disp_cc_pll0_out_even.clkr.hw }, +}; + +static const struct parent_map disp_cc_parent_map_4[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 4 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_4[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "gcc_disp_gpll0_clk_src" }, +}; + +static const struct parent_map disp_cc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_DSICLK, 1 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_5[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "dsi0_phy_pll_out_dsiclk" }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_ahb_clk_src = { + .cmd_rcgr = 0x22bc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_4, + .freq_tbl = ftbl_disp_cc_mdss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_ahb_clk_src", + .parent_data = disp_cc_parent_data_4, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_byte0_clk_src = { + .cmd_rcgr = 0x2110, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_dp_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_dp_aux_clk_src = { + .cmd_rcgr = 0x21dc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_crypto_clk_src = { + .cmd_rcgr = 0x2194, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_crypto_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_link_clk_src = { + .cmd_rcgr = 0x2178, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_pixel_clk_src = { + .cmd_rcgr = 0x21ac, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_esc0_clk_src = { + .cmd_rcgr = 0x2148, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc0_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_mdp_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F(345000000, P_DISP_CC_PLL0_OUT_MAIN, 4, 0, 0), + F(460000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_mdp_clk_src = { + .cmd_rcgr = 0x20c8, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { + .cmd_rcgr = 0x2098, + .mnd_width = 8, + .hid_width = 5, + .parent_map = disp_cc_parent_map_5, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk0_clk_src", + .parent_data = disp_cc_parent_data_5, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_5), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_pixel_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_rot_clk_src = { + .cmd_rcgr = 0x20e0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rot_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_vsync_clk_src = { + .cmd_rcgr = 0x20f8, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_vsync_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_branch disp_cc_mdss_ahb_clk = { + .halt_reg = 0x2080, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_clk = { + .halt_reg = 0x2028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { + .reg = 0x2128, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte0_clk_src.clkr.hw + }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_dp_link_div_clk_src = { + .reg = 0x2190, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dp_link_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link_clk_src.clkr.hw + }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_intf_clk = { + .halt_reg = 0x202c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x202c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_intf_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_byte0_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_aux_clk = { + .halt_reg = 0x2054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_crypto_clk = { + .halt_reg = 0x2048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_crypto_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_crypto_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_link_clk = { + .halt_reg = 0x2040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_link_intf_clk = { + .halt_reg = 0x2044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_intf_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_pixel_clk = { + .halt_reg = 0x204c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x204c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_dp_pixel_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_esc0_clk = { + .halt_reg = 0x2038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_esc0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_clk = { + .halt_reg = 0x200c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x200c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_lut_clk = { + .halt_reg = 0x201c, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_lut_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_non_gdsc_ahb_clk = { + .halt_reg = 0x4004, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_non_gdsc_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_pclk0_clk = { + .halt_reg = 0x2004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_pclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rot_clk = { + .halt_reg = 0x2014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rot_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_rot_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rscc_ahb_clk = { + .halt_reg = 0x400c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x400c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rscc_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rscc_vsync_clk = { + .halt_reg = 0x4008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rscc_vsync_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_vsync_clk = { + .halt_reg = 0x2024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_vsync_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x3000, + .pd = { + .name = "mdss_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, +}; + +static struct gdsc *disp_cc_sc7180_gdscs[] = { + [MDSS_GDSC] = &mdss_gdsc, +}; + +static struct clk_regmap *disp_cc_sc7180_clocks[] = { + [DISP_CC_MDSS_AHB_CLK] = &disp_cc_mdss_ahb_clk.clkr, + [DISP_CC_MDSS_AHB_CLK_SRC] = &disp_cc_mdss_ahb_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_CLK] = &disp_cc_mdss_byte0_clk.clkr, + [DISP_CC_MDSS_BYTE0_CLK_SRC] = &disp_cc_mdss_byte0_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_DIV_CLK_SRC] = &disp_cc_mdss_byte0_div_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_INTF_CLK] = &disp_cc_mdss_byte0_intf_clk.clkr, + [DISP_CC_MDSS_DP_AUX_CLK] = &disp_cc_mdss_dp_aux_clk.clkr, + [DISP_CC_MDSS_DP_AUX_CLK_SRC] = &disp_cc_mdss_dp_aux_clk_src.clkr, + [DISP_CC_MDSS_DP_CRYPTO_CLK] = &disp_cc_mdss_dp_crypto_clk.clkr, + [DISP_CC_MDSS_DP_CRYPTO_CLK_SRC] = &disp_cc_mdss_dp_crypto_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK_CLK] = &disp_cc_mdss_dp_link_clk.clkr, + [DISP_CC_MDSS_DP_LINK_CLK_SRC] = &disp_cc_mdss_dp_link_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK_DIV_CLK_SRC] = + &disp_cc_mdss_dp_link_div_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK_INTF_CLK] = &disp_cc_mdss_dp_link_intf_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL_CLK] = &disp_cc_mdss_dp_pixel_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL_CLK_SRC] = &disp_cc_mdss_dp_pixel_clk_src.clkr, + [DISP_CC_MDSS_ESC0_CLK] = &disp_cc_mdss_esc0_clk.clkr, + [DISP_CC_MDSS_ESC0_CLK_SRC] = &disp_cc_mdss_esc0_clk_src.clkr, + [DISP_CC_MDSS_MDP_CLK] = &disp_cc_mdss_mdp_clk.clkr, + [DISP_CC_MDSS_MDP_CLK_SRC] = &disp_cc_mdss_mdp_clk_src.clkr, + [DISP_CC_MDSS_MDP_LUT_CLK] = &disp_cc_mdss_mdp_lut_clk.clkr, + [DISP_CC_MDSS_NON_GDSC_AHB_CLK] = &disp_cc_mdss_non_gdsc_ahb_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK] = &disp_cc_mdss_pclk0_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK_SRC] = &disp_cc_mdss_pclk0_clk_src.clkr, + [DISP_CC_MDSS_ROT_CLK] = &disp_cc_mdss_rot_clk.clkr, + [DISP_CC_MDSS_ROT_CLK_SRC] = &disp_cc_mdss_rot_clk_src.clkr, + [DISP_CC_MDSS_RSCC_AHB_CLK] = &disp_cc_mdss_rscc_ahb_clk.clkr, + [DISP_CC_MDSS_RSCC_VSYNC_CLK] = &disp_cc_mdss_rscc_vsync_clk.clkr, + [DISP_CC_MDSS_VSYNC_CLK] = &disp_cc_mdss_vsync_clk.clkr, + [DISP_CC_MDSS_VSYNC_CLK_SRC] = &disp_cc_mdss_vsync_clk_src.clkr, + [DISP_CC_PLL0] = &disp_cc_pll0.clkr, + [DISP_CC_PLL0_OUT_EVEN] = &disp_cc_pll0_out_even.clkr, +}; + +static const struct regmap_config disp_cc_sc7180_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x10000, + .fast_io = true, +}; + +static const struct qcom_cc_desc disp_cc_sc7180_desc = { + .config = &disp_cc_sc7180_regmap_config, + .clks = disp_cc_sc7180_clocks, + .num_clks = ARRAY_SIZE(disp_cc_sc7180_clocks), + .gdscs = disp_cc_sc7180_gdscs, + .num_gdscs = ARRAY_SIZE(disp_cc_sc7180_gdscs), +}; + +static const struct of_device_id disp_cc_sc7180_match_table[] = { + { .compatible = "qcom,sc7180-dispcc" }, + { } +}; +MODULE_DEVICE_TABLE(of, disp_cc_sc7180_match_table); + +static int disp_cc_sc7180_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + struct alpha_pll_config disp_cc_pll_config = {}; + + regmap = qcom_cc_map(pdev, &disp_cc_sc7180_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* 1380MHz configuration */ + disp_cc_pll_config.l = 0x47; + disp_cc_pll_config.alpha = 0xe000; + disp_cc_pll_config.user_ctl_val = 0x00000001; + disp_cc_pll_config.user_ctl_hi_val = 0x00004805; + + clk_fabia_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll_config); + + return qcom_cc_really_probe(pdev, &disp_cc_sc7180_desc, regmap); +} + +static struct platform_driver disp_cc_sc7180_driver = { + .probe = disp_cc_sc7180_probe, + .driver = { + .name = "sc7180-dispcc", + .of_match_table = disp_cc_sc7180_match_table, + }, +}; + +static int __init disp_cc_sc7180_init(void) +{ + return platform_driver_register(&disp_cc_sc7180_driver); +} +subsys_initcall(disp_cc_sc7180_init); + +static void __exit disp_cc_sc7180_exit(void) +{ + platform_driver_unregister(&disp_cc_sc7180_driver); +} +module_exit(disp_cc_sc7180_exit); + +MODULE_DESCRIPTION("QTI DISP_CC SC7180 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/dispcc-sdm845.c b/drivers/clk/qcom/dispcc-sdm845.c index 0cc4909b5dbe..5c932cd17b14 100644 --- a/drivers/clk/qcom/dispcc-sdm845.c +++ b/drivers/clk/qcom/dispcc-sdm845.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2018, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */ #include <linux/clk-provider.h> @@ -29,6 +29,8 @@ enum { P_DSI1_PHY_PLL_OUT_DSICLK, P_GPLL0_OUT_MAIN, P_GPLL0_OUT_MAIN_DIV, + P_DP_PHY_PLL_LINK_CLK, + P_DP_PHY_PLL_VCO_DIV_CLK, }; static const struct parent_map disp_cc_parent_map_0[] = { @@ -45,6 +47,20 @@ static const char * const disp_cc_parent_names_0[] = { "core_bi_pll_test_se", }; +static const struct parent_map disp_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_DP_PHY_PLL_LINK_CLK, 1 }, + { P_DP_PHY_PLL_VCO_DIV_CLK, 2 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const char * const disp_cc_parent_names_1[] = { + "bi_tcxo", + "dp_link_clk_divsel_ten", + "dp_vco_divided_clk_src_mux", + "core_bi_pll_test_se", +}; + static const struct parent_map disp_cc_parent_map_2[] = { { P_BI_TCXO, 0 }, { P_CORE_BI_PLL_TEST_SE, 7 }, @@ -128,6 +144,81 @@ static struct clk_rcg2 disp_cc_mdss_byte1_clk_src = { }, }; +static const struct freq_tbl ftbl_disp_cc_mdss_dp_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_dp_aux_clk_src = { + .cmd_rcgr = 0x219c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux_clk_src", + .parent_names = disp_cc_parent_names_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_crypto_clk_src = { + .cmd_rcgr = 0x2154, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_crypto_clk_src", + .parent_names = disp_cc_parent_names_1, + .num_parents = 4, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_link_clk_src = { + .cmd_rcgr = 0x2138, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_clk_src", + .parent_names = disp_cc_parent_names_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_pixel1_clk_src = { + .cmd_rcgr = 0x2184, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel1_clk_src", + .parent_names = disp_cc_parent_names_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dp_pixel_clk_src = { + .cmd_rcgr = 0x216c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel_clk_src", + .parent_names = disp_cc_parent_names_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + static const struct freq_tbl ftbl_disp_cc_mdss_esc0_clk_src[] = { F(19200000, P_BI_TCXO, 1, 0, 0), { } @@ -391,6 +482,114 @@ static struct clk_branch disp_cc_mdss_byte1_intf_clk = { }, }; +static struct clk_branch disp_cc_mdss_dp_aux_clk = { + .halt_reg = 0x2054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_aux_clk", + .parent_names = (const char *[]){ + "disp_cc_mdss_dp_aux_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_crypto_clk = { + .halt_reg = 0x2048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_crypto_clk", + .parent_names = (const char *[]){ + "disp_cc_mdss_dp_crypto_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_link_clk = { + .halt_reg = 0x2040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_clk", + .parent_names = (const char *[]){ + "disp_cc_mdss_dp_link_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* reset state of disp_cc_mdss_dp_link_div_clk_src divider is 0x3 (div 4) */ +static struct clk_branch disp_cc_mdss_dp_link_intf_clk = { + .halt_reg = 0x2044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_link_intf_clk", + .parent_names = (const char *[]){ + "disp_cc_mdss_dp_link_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_pixel1_clk = { + .halt_reg = 0x2050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel1_clk", + .parent_names = (const char *[]){ + "disp_cc_mdss_dp_pixel1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dp_pixel_clk = { + .halt_reg = 0x204c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x204c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_dp_pixel_clk", + .parent_names = (const char *[]){ + "disp_cc_mdss_dp_pixel_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch disp_cc_mdss_esc0_clk = { .halt_reg = 0x2038, .halt_check = BRANCH_HALT, @@ -589,6 +788,19 @@ static struct clk_regmap *disp_cc_sdm845_clocks[] = { [DISP_CC_MDSS_BYTE1_INTF_CLK] = &disp_cc_mdss_byte1_intf_clk.clkr, [DISP_CC_MDSS_BYTE1_DIV_CLK_SRC] = &disp_cc_mdss_byte1_div_clk_src.clkr, + [DISP_CC_MDSS_DP_AUX_CLK] = &disp_cc_mdss_dp_aux_clk.clkr, + [DISP_CC_MDSS_DP_AUX_CLK_SRC] = &disp_cc_mdss_dp_aux_clk_src.clkr, + [DISP_CC_MDSS_DP_CRYPTO_CLK] = &disp_cc_mdss_dp_crypto_clk.clkr, + [DISP_CC_MDSS_DP_CRYPTO_CLK_SRC] = + &disp_cc_mdss_dp_crypto_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK_CLK] = &disp_cc_mdss_dp_link_clk.clkr, + [DISP_CC_MDSS_DP_LINK_CLK_SRC] = &disp_cc_mdss_dp_link_clk_src.clkr, + [DISP_CC_MDSS_DP_LINK_INTF_CLK] = &disp_cc_mdss_dp_link_intf_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL1_CLK] = &disp_cc_mdss_dp_pixel1_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL1_CLK_SRC] = + &disp_cc_mdss_dp_pixel1_clk_src.clkr, + [DISP_CC_MDSS_DP_PIXEL_CLK] = &disp_cc_mdss_dp_pixel_clk.clkr, + [DISP_CC_MDSS_DP_PIXEL_CLK_SRC] = &disp_cc_mdss_dp_pixel_clk_src.clkr, [DISP_CC_MDSS_ESC0_CLK] = &disp_cc_mdss_esc0_clk.clkr, [DISP_CC_MDSS_ESC0_CLK_SRC] = &disp_cc_mdss_esc0_clk_src.clkr, [DISP_CC_MDSS_ESC1_CLK] = &disp_cc_mdss_esc1_clk.clkr, diff --git a/drivers/clk/qcom/gcc-ipq6018.c b/drivers/clk/qcom/gcc-ipq6018.c new file mode 100644 index 000000000000..3f9c2f61a5d9 --- /dev/null +++ b/drivers/clk/qcom/gcc-ipq6018.c @@ -0,0 +1,4635 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018, The Linux Foundation. All rights reserved. + */ + +#include <linux/kernel.h> +#include <linux/err.h> +#include <linux/platform_device.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/clk-provider.h> +#include <linux/regmap.h> + +#include <linux/reset-controller.h> +#include <dt-bindings/clock/qcom,gcc-ipq6018.h> +#include <dt-bindings/reset/qcom,gcc-ipq6018.h> + +#include "common.h" +#include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "clk-alpha-pll.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "reset.h" + +#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) } + +enum { + P_XO, + P_BIAS_PLL, + P_UNIPHY0_RX, + P_UNIPHY0_TX, + P_UNIPHY1_RX, + P_BIAS_PLL_NSS_NOC, + P_UNIPHY1_TX, + P_PCIE20_PHY0_PIPE, + P_USB3PHY_0_PIPE, + P_GPLL0, + P_GPLL0_DIV2, + P_GPLL2, + P_GPLL4, + P_GPLL6, + P_SLEEP_CLK, + P_UBI32_PLL, + P_NSS_CRYPTO_PLL, + P_PI_SLEEP, +}; + +static struct clk_alpha_pll gpll0_main = { + .offset = 0x21000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x0b000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0_main", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static struct clk_fixed_factor gpll0_out_main_div2 = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "gpll0_out_main_div2", + .parent_hws = (const struct clk_hw *[]){ + &gpll0_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_alpha_pll_postdiv gpll0 = { + .offset = 0x21000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .width = 4, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_hws = (const struct clk_hw *[]){ + &gpll0_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_ro_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct clk_parent_data gcc_xo_gpll0_gpll0_out_main_div2[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw}, + { .hw = &gpll0_out_main_div2.hw}, +}; + +static const struct parent_map gcc_xo_gpll0_gpll0_out_main_div2_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL0_DIV2, 4 }, +}; + +static struct clk_alpha_pll ubi32_pll_main = { + .offset = 0x25000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA], + .flags = SUPPORTS_DYNAMIC_UPDATE, + .clkr = { + .enable_reg = 0x0b000, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "ubi32_pll_main", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_huayra_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv ubi32_pll = { + .offset = 0x25000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA], + .width = 2, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ubi32_pll", + .parent_hws = (const struct clk_hw *[]){ + &ubi32_pll_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_ro_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_alpha_pll gpll6_main = { + .offset = 0x37000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO], + .clkr = { + .enable_reg = 0x0b000, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gpll6_main", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv gpll6 = { + .offset = 0x37000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO], + .width = 2, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll6", + .parent_hws = (const struct clk_hw *[]){ + &gpll6_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_ro_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_alpha_pll gpll4_main = { + .offset = 0x24000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x0b000, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gpll4_main", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv gpll4 = { + .offset = 0x24000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .width = 4, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll4", + .parent_hws = (const struct clk_hw *[]){ + &gpll4_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_ro_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_pcnoc_bfdcd_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0, 16, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 pcnoc_bfdcd_clk_src = { + .cmd_rcgr = 0x27000, + .freq_tbl = ftbl_pcnoc_bfdcd_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcnoc_bfdcd_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_alpha_pll gpll2_main = { + .offset = 0x4a000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x0b000, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gpll2_main", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv gpll2 = { + .offset = 0x4a000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .width = 4, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll2", + .parent_hws = (const struct clk_hw *[]){ + &gpll2_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_ro_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_alpha_pll nss_crypto_pll_main = { + .offset = 0x22000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x0b000, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "nss_crypto_pll_main", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv nss_crypto_pll = { + .offset = 0x22000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .width = 4, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_crypto_pll", + .parent_hws = (const struct clk_hw *[]){ + &nss_crypto_pll_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_ro_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_qdss_tsctr_clk_src[] = { + F(160000000, P_GPLL0_DIV2, 2.5, 0, 0), + F(320000000, P_GPLL0, 2.5, 0, 0), + F(600000000, P_GPLL4, 2, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_gpll4_gpll0_gpll6_gpll0_div2[] = { + { .fw_name = "xo" }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll6.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, +}; + +static const struct parent_map gcc_xo_gpll4_gpll0_gpll6_gpll0_div2_map[] = { + { P_XO, 0 }, + { P_GPLL4, 1 }, + { P_GPLL0, 2 }, + { P_GPLL6, 3 }, + { P_GPLL0_DIV2, 4 }, +}; + +static struct clk_rcg2 qdss_tsctr_clk_src = { + .cmd_rcgr = 0x29064, + .freq_tbl = ftbl_qdss_tsctr_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll4_gpll0_gpll6_gpll0_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "qdss_tsctr_clk_src", + .parent_data = gcc_xo_gpll4_gpll0_gpll6_gpll0_div2, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_fixed_factor qdss_dap_sync_clk_src = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "qdss_dap_sync_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &qdss_tsctr_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + }, +}; + +static const struct freq_tbl ftbl_qdss_at_clk_src[] = { + F(66670000, P_GPLL0_DIV2, 6, 0, 0), + F(240000000, P_GPLL4, 5, 0, 0), + { } +}; + +static struct clk_rcg2 qdss_at_clk_src = { + .cmd_rcgr = 0x2900c, + .freq_tbl = ftbl_qdss_at_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll4_gpll0_gpll6_gpll0_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "qdss_at_clk_src", + .parent_data = gcc_xo_gpll4_gpll0_gpll6_gpll0_div2, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_fixed_factor qdss_tsctr_div2_clk_src = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "qdss_tsctr_div2_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &qdss_tsctr_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_fixed_factor_ops, + }, +}; + +static const struct freq_tbl ftbl_nss_ppe_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(300000000, P_BIAS_PLL, 1, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_bias_gpll0_gpll4_nss_ubi32[] = { + { .fw_name = "xo" }, + { .fw_name = "bias_pll_cc_clk" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll4.clkr.hw }, + { .hw = &nss_crypto_pll.clkr.hw }, + { .hw = &ubi32_pll.clkr.hw }, +}; + +static const struct parent_map gcc_xo_bias_gpll0_gpll4_nss_ubi32_map[] = { + { P_XO, 0 }, + { P_BIAS_PLL, 1 }, + { P_GPLL0, 2 }, + { P_GPLL4, 3 }, + { P_NSS_CRYPTO_PLL, 4 }, + { P_UBI32_PLL, 5 }, +}; + +static struct clk_rcg2 nss_ppe_clk_src = { + .cmd_rcgr = 0x68080, + .freq_tbl = ftbl_nss_ppe_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_bias_gpll0_gpll4_nss_ubi32_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_ppe_clk_src", + .parent_data = gcc_xo_bias_gpll0_gpll4_nss_ubi32, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_xo_clk_src = { + .halt_reg = 0x30018, + .clkr = { + .enable_reg = 0x30018, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_xo_clk_src", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_nss_ce_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_gpll0[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, +}; + +static struct clk_rcg2 nss_ce_clk_src = { + .cmd_rcgr = 0x68098, + .freq_tbl = ftbl_nss_ce_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_ce_clk_src", + .parent_data = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_sleep_clk_src = { + .halt_reg = 0x30000, + .clkr = { + .enable_reg = 0x30000, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sleep_clk_src", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "sleep_clk", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_snoc_nssnoc_bfdcd_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0_DIV2, 8, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(133333333, P_GPLL0, 6, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(266666667, P_GPLL0, 3, 0, 0), + { } +}; + +static const struct clk_parent_data + gcc_xo_gpll0_gpll6_gpll0_out_main_div2[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll6.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_gpll6_gpll0_out_main_div2_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL6, 2 }, + { P_GPLL0_DIV2, 3 }, +}; + +static struct clk_rcg2 snoc_nssnoc_bfdcd_clk_src = { + .cmd_rcgr = 0x76054, + .freq_tbl = ftbl_snoc_nssnoc_bfdcd_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll6_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "snoc_nssnoc_bfdcd_clk_src", + .parent_data = gcc_xo_gpll0_gpll6_gpll0_out_main_div2, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_apss_ahb_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(25000000, P_GPLL0_DIV2, 16, 0, 0), + F(50000000, P_GPLL0, 16, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 apss_ahb_clk_src = { + .cmd_rcgr = 0x46000, + .freq_tbl = ftbl_apss_ahb_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "apss_ahb_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_nss_port5_rx_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(25000000, P_UNIPHY1_RX, 12.5, 0, 0), + F(25000000, P_UNIPHY0_RX, 5, 0, 0), + F(78125000, P_UNIPHY1_RX, 4, 0, 0), + F(125000000, P_UNIPHY1_RX, 2.5, 0, 0), + F(125000000, P_UNIPHY0_RX, 1, 0, 0), + F(156250000, P_UNIPHY1_RX, 2, 0, 0), + F(312500000, P_UNIPHY1_RX, 1, 0, 0), + { } +}; + +static const struct clk_parent_data +gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias[] = { + { .fw_name = "xo" }, + { .fw_name = "uniphy0_gcc_rx_clk" }, + { .fw_name = "uniphy0_gcc_tx_clk" }, + { .fw_name = "uniphy1_gcc_rx_clk" }, + { .fw_name = "uniphy1_gcc_tx_clk" }, + { .hw = &ubi32_pll.clkr.hw }, + { .fw_name = "bias_pll_cc_clk" }, +}; + +static const struct parent_map +gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map[] = { + { P_XO, 0 }, + { P_UNIPHY0_RX, 1 }, + { P_UNIPHY0_TX, 2 }, + { P_UNIPHY1_RX, 3 }, + { P_UNIPHY1_TX, 4 }, + { P_UBI32_PLL, 5 }, + { P_BIAS_PLL, 6 }, +}; + +static struct clk_rcg2 nss_port5_rx_clk_src = { + .cmd_rcgr = 0x68060, + .freq_tbl = ftbl_nss_port5_rx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port5_rx_clk_src", + .parent_data = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_nss_port5_tx_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(25000000, P_UNIPHY1_TX, 12.5, 0, 0), + F(25000000, P_UNIPHY0_TX, 5, 0, 0), + F(78125000, P_UNIPHY1_TX, 4, 0, 0), + F(125000000, P_UNIPHY1_TX, 2.5, 0, 0), + F(125000000, P_UNIPHY0_TX, 1, 0, 0), + F(156250000, P_UNIPHY1_TX, 2, 0, 0), + F(312500000, P_UNIPHY1_TX, 1, 0, 0), + { } +}; + +static const struct clk_parent_data +gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias[] = { + { .fw_name = "xo" }, + { .fw_name = "uniphy0_gcc_tx_clk" }, + { .fw_name = "uniphy0_gcc_rx_clk" }, + { .fw_name = "uniphy1_gcc_tx_clk" }, + { .fw_name = "uniphy1_gcc_rx_clk" }, + { .hw = &ubi32_pll.clkr.hw }, + { .fw_name = "bias_pll_cc_clk" }, +}; + +static const struct parent_map +gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map[] = { + { P_XO, 0 }, + { P_UNIPHY0_TX, 1 }, + { P_UNIPHY0_RX, 2 }, + { P_UNIPHY1_TX, 3 }, + { P_UNIPHY1_RX, 4 }, + { P_UBI32_PLL, 5 }, + { P_BIAS_PLL, 6 }, +}; + +static struct clk_rcg2 nss_port5_tx_clk_src = { + .cmd_rcgr = 0x68068, + .freq_tbl = ftbl_nss_port5_tx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port5_tx_clk_src", + .parent_data = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_pcie_axi_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(240000000, P_GPLL4, 5, 0, 0), + { } +}; + +static const struct freq_tbl ftbl_pcie_rchng_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_gpll0_gpll4[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll4.clkr.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_gpll4_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL4, 2 }, +}; + +static struct clk_rcg2 pcie0_axi_clk_src = { + .cmd_rcgr = 0x75054, + .freq_tbl = ftbl_pcie_axi_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll4_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcie0_axi_clk_src", + .parent_data = gcc_xo_gpll0_gpll4, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_usb0_master_clk_src[] = { + F(80000000, P_GPLL0_DIV2, 5, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(133330000, P_GPLL0, 6, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_gpll0_out_main_div2_gpll0[] = { + { .fw_name = "xo" }, + { .hw = &gpll0_out_main_div2.hw }, + { .hw = &gpll0.clkr.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_out_main_div2_gpll0_map[] = { + { P_XO, 0 }, + { P_GPLL0_DIV2, 2 }, + { P_GPLL0, 1 }, +}; + +static struct clk_rcg2 usb0_master_clk_src = { + .cmd_rcgr = 0x3e00c, + .freq_tbl = ftbl_usb0_master_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_out_main_div2_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb0_master_clk_src", + .parent_data = gcc_xo_gpll0_out_main_div2_gpll0, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div apss_ahb_postdiv_clk_src = { + .reg = 0x46018, + .shift = 4, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "apss_ahb_postdiv_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &apss_ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, + }, +}; + +static struct clk_fixed_factor gcc_xo_div4_clk_src = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "gcc_xo_div4_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &gcc_xo_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_nss_port1_rx_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(25000000, P_UNIPHY0_RX, 5, 0, 0), + F(125000000, P_UNIPHY0_RX, 1, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_uniphy0_rx_tx_ubi32_bias[] = { + { .fw_name = "xo" }, + { .fw_name = "uniphy0_gcc_rx_clk" }, + { .fw_name = "uniphy0_gcc_tx_clk" }, + { .hw = &ubi32_pll.clkr.hw }, + { .fw_name = "bias_pll_cc_clk" }, +}; + +static const struct parent_map gcc_xo_uniphy0_rx_tx_ubi32_bias_map[] = { + { P_XO, 0 }, + { P_UNIPHY0_RX, 1 }, + { P_UNIPHY0_TX, 2 }, + { P_UBI32_PLL, 5 }, + { P_BIAS_PLL, 6 }, +}; + +static struct clk_rcg2 nss_port1_rx_clk_src = { + .cmd_rcgr = 0x68020, + .freq_tbl = ftbl_nss_port1_rx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port1_rx_clk_src", + .parent_data = gcc_xo_uniphy0_rx_tx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_nss_port1_tx_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(25000000, P_UNIPHY0_TX, 5, 0, 0), + F(125000000, P_UNIPHY0_TX, 1, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_uniphy0_tx_rx_ubi32_bias[] = { + { .fw_name = "xo" }, + { .fw_name = "uniphy0_gcc_tx_clk" }, + { .fw_name = "uniphy0_gcc_rx_clk" }, + { .hw = &ubi32_pll.clkr.hw }, + { .fw_name = "bias_pll_cc_clk" }, +}; + +static const struct parent_map gcc_xo_uniphy0_tx_rx_ubi32_bias_map[] = { + { P_XO, 0 }, + { P_UNIPHY0_TX, 1 }, + { P_UNIPHY0_RX, 2 }, + { P_UBI32_PLL, 5 }, + { P_BIAS_PLL, 6 }, +}; + +static struct clk_rcg2 nss_port1_tx_clk_src = { + .cmd_rcgr = 0x68028, + .freq_tbl = ftbl_nss_port1_tx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port1_tx_clk_src", + .parent_data = gcc_xo_uniphy0_tx_rx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 nss_port2_rx_clk_src = { + .cmd_rcgr = 0x68030, + .freq_tbl = ftbl_nss_port1_rx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port2_rx_clk_src", + .parent_data = gcc_xo_uniphy0_rx_tx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 nss_port2_tx_clk_src = { + .cmd_rcgr = 0x68038, + .freq_tbl = ftbl_nss_port1_tx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port2_tx_clk_src", + .parent_data = gcc_xo_uniphy0_tx_rx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 nss_port3_rx_clk_src = { + .cmd_rcgr = 0x68040, + .freq_tbl = ftbl_nss_port1_rx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port3_rx_clk_src", + .parent_data = gcc_xo_uniphy0_rx_tx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 nss_port3_tx_clk_src = { + .cmd_rcgr = 0x68048, + .freq_tbl = ftbl_nss_port1_tx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port3_tx_clk_src", + .parent_data = gcc_xo_uniphy0_tx_rx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 nss_port4_rx_clk_src = { + .cmd_rcgr = 0x68050, + .freq_tbl = ftbl_nss_port1_rx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port4_rx_clk_src", + .parent_data = gcc_xo_uniphy0_rx_tx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 nss_port4_tx_clk_src = { + .cmd_rcgr = 0x68058, + .freq_tbl = ftbl_nss_port1_tx_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_port4_tx_clk_src", + .parent_data = gcc_xo_uniphy0_tx_rx_ubi32_bias, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div nss_port5_rx_div_clk_src = { + .reg = 0x68440, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port5_rx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_rx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port5_tx_div_clk_src = { + .reg = 0x68444, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port5_tx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_tx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl ftbl_apss_axi_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(100000000, P_GPLL0_DIV2, 4, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(308570000, P_GPLL6, 3.5, 0, 0), + F(400000000, P_GPLL0, 2, 0, 0), + F(533000000, P_GPLL0, 1.5, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_gpll0_gpll6_ubi32_gpll0_div2[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll6.clkr.hw }, + { .hw = &ubi32_pll.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, +}; + +static const struct parent_map +gcc_xo_gpll0_gpll6_ubi32_gpll0_div2_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL6, 2 }, + { P_UBI32_PLL, 3 }, + { P_GPLL0_DIV2, 6 }, +}; + +static struct clk_rcg2 apss_axi_clk_src = { + .cmd_rcgr = 0x38048, + .freq_tbl = ftbl_apss_axi_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll6_ubi32_gpll0_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "apss_axi_clk_src", + .parent_data = gcc_xo_gpll0_gpll6_ubi32_gpll0_div2, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_nss_crypto_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(300000000, P_NSS_CRYPTO_PLL, 2, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_nss_crypto_pll_gpll0[] = { + { .fw_name = "xo" }, + { .hw = &nss_crypto_pll.clkr.hw }, + { .hw = &gpll0.clkr.hw }, +}; + +static const struct parent_map gcc_xo_nss_crypto_pll_gpll0_map[] = { + { P_XO, 0 }, + { P_NSS_CRYPTO_PLL, 1 }, + { P_GPLL0, 2 }, +}; + +static struct clk_rcg2 nss_crypto_clk_src = { + .cmd_rcgr = 0x68144, + .freq_tbl = ftbl_nss_crypto_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_nss_crypto_pll_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_crypto_clk_src", + .parent_data = gcc_xo_nss_crypto_pll_gpll0, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div nss_port1_rx_div_clk_src = { + .reg = 0x68400, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port1_rx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port1_rx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port1_tx_div_clk_src = { + .reg = 0x68404, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port1_tx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port1_tx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port2_rx_div_clk_src = { + .reg = 0x68410, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port2_rx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port2_rx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port2_tx_div_clk_src = { + .reg = 0x68414, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port2_tx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port2_tx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port3_rx_div_clk_src = { + .reg = 0x68420, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port3_rx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port3_rx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port3_tx_div_clk_src = { + .reg = 0x68424, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port3_tx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port3_tx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port4_rx_div_clk_src = { + .reg = 0x68430, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port4_rx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port4_rx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_regmap_div nss_port4_tx_div_clk_src = { + .reg = 0x68434, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_port4_tx_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_port4_tx_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl ftbl_nss_ubi_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(149760000, P_UBI32_PLL, 10, 0, 0), + F(187200000, P_UBI32_PLL, 8, 0, 0), + F(249600000, P_UBI32_PLL, 6, 0, 0), + F(374400000, P_UBI32_PLL, 4, 0, 0), + F(748800000, P_UBI32_PLL, 2, 0, 0), + F(1497600000, P_UBI32_PLL, 1, 0, 0), + { } +}; + +static const struct clk_parent_data + gcc_xo_ubi32_pll_gpll0_gpll2_gpll4_gpll6[] = { + { .fw_name = "xo" }, + { .hw = &ubi32_pll.clkr.hw }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll2.clkr.hw }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll6.clkr.hw }, +}; + +static const struct parent_map gcc_xo_ubi32_gpll0_gpll2_gpll4_gpll6_map[] = { + { P_XO, 0 }, + { P_UBI32_PLL, 1 }, + { P_GPLL0, 2 }, + { P_GPLL2, 3 }, + { P_GPLL4, 4 }, + { P_GPLL6, 5 }, +}; + +static struct clk_rcg2 nss_ubi0_clk_src = { + .cmd_rcgr = 0x68104, + .freq_tbl = ftbl_nss_ubi_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_ubi32_gpll0_gpll2_gpll4_gpll6_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "nss_ubi0_clk_src", + .parent_data = gcc_xo_ubi32_pll_gpll0_gpll2_gpll4_gpll6, + .num_parents = 6, + .ops = &clk_rcg2_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_adss_pwm_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 adss_pwm_clk_src = { + .cmd_rcgr = 0x1c008, + .freq_tbl = ftbl_adss_pwm_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "adss_pwm_clk_src", + .parent_data = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_blsp1_qup_i2c_apps_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(25000000, P_GPLL0_DIV2, 16, 0, 0), + F(50000000, P_GPLL0, 16, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = { + .cmd_rcgr = 0x0200c, + .freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup1_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_blsp1_qup_spi_apps_clk_src[] = { + F(960000, P_XO, 10, 2, 5), + F(4800000, P_XO, 5, 0, 0), + F(9600000, P_XO, 2, 4, 5), + F(12500000, P_GPLL0_DIV2, 16, 1, 2), + F(16000000, P_GPLL0, 10, 1, 5), + F(24000000, P_XO, 1, 0, 0), + F(25000000, P_GPLL0, 16, 1, 2), + F(50000000, P_GPLL0, 16, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = { + .cmd_rcgr = 0x02024, + .freq_tbl = ftbl_blsp1_qup_spi_apps_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup1_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = { + .cmd_rcgr = 0x03000, + .freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup2_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = { + .cmd_rcgr = 0x03014, + .freq_tbl = ftbl_blsp1_qup_spi_apps_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup2_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = { + .cmd_rcgr = 0x04000, + .freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup3_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = { + .cmd_rcgr = 0x04014, + .freq_tbl = ftbl_blsp1_qup_spi_apps_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup3_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = { + .cmd_rcgr = 0x05000, + .freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup4_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = { + .cmd_rcgr = 0x05014, + .freq_tbl = ftbl_blsp1_qup_spi_apps_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup4_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = { + .cmd_rcgr = 0x06000, + .freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup5_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = { + .cmd_rcgr = 0x06014, + .freq_tbl = ftbl_blsp1_qup_spi_apps_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup5_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = { + .cmd_rcgr = 0x07000, + .freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup6_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = { + .cmd_rcgr = 0x07014, + .freq_tbl = ftbl_blsp1_qup_spi_apps_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup6_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_blsp1_uart_apps_clk_src[] = { + F(3686400, P_GPLL0_DIV2, 1, 144, 15625), + F(7372800, P_GPLL0_DIV2, 1, 288, 15625), + F(14745600, P_GPLL0_DIV2, 1, 576, 15625), + F(16000000, P_GPLL0_DIV2, 5, 1, 5), + F(24000000, P_XO, 1, 0, 0), + F(24000000, P_GPLL0, 1, 3, 100), + F(25000000, P_GPLL0, 16, 1, 2), + F(32000000, P_GPLL0, 1, 1, 25), + F(40000000, P_GPLL0, 1, 1, 20), + F(46400000, P_GPLL0, 1, 29, 500), + F(48000000, P_GPLL0, 1, 3, 50), + F(51200000, P_GPLL0, 1, 8, 125), + F(56000000, P_GPLL0, 1, 7, 100), + F(58982400, P_GPLL0, 1, 1152, 15625), + F(60000000, P_GPLL0, 1, 3, 40), + F(64000000, P_GPLL0, 12.5, 1, 1), + { } +}; + +static struct clk_rcg2 blsp1_uart1_apps_clk_src = { + .cmd_rcgr = 0x02044, + .freq_tbl = ftbl_blsp1_uart_apps_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart1_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart2_apps_clk_src = { + .cmd_rcgr = 0x03034, + .freq_tbl = ftbl_blsp1_uart_apps_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart2_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart3_apps_clk_src = { + .cmd_rcgr = 0x04034, + .freq_tbl = ftbl_blsp1_uart_apps_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart3_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart4_apps_clk_src = { + .cmd_rcgr = 0x05034, + .freq_tbl = ftbl_blsp1_uart_apps_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart4_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart5_apps_clk_src = { + .cmd_rcgr = 0x06034, + .freq_tbl = ftbl_blsp1_uart_apps_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart5_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart6_apps_clk_src = { + .cmd_rcgr = 0x07034, + .freq_tbl = ftbl_blsp1_uart_apps_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart6_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_crypto_clk_src[] = { + F(40000000, P_GPLL0_DIV2, 10, 0, 0), + F(80000000, P_GPLL0, 10, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + { } +}; + +static struct clk_rcg2 crypto_clk_src = { + .cmd_rcgr = 0x16004, + .freq_tbl = ftbl_crypto_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "crypto_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gp_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0_DIV2, 8, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(266666666, P_GPLL0, 3, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_gpll0_gpll6_gpll0_sleep_clk[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll6.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, + { .fw_name = "sleep_clk" }, +}; + +static const struct parent_map gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL6, 2 }, + { P_GPLL0_DIV2, 4 }, + { P_SLEEP_CLK, 6 }, +}; + +static struct clk_rcg2 gp1_clk_src = { + .cmd_rcgr = 0x08004, + .freq_tbl = ftbl_gp_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp1_clk_src", + .parent_data = gcc_xo_gpll0_gpll6_gpll0_sleep_clk, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gp2_clk_src = { + .cmd_rcgr = 0x09004, + .freq_tbl = ftbl_gp_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp2_clk_src", + .parent_data = gcc_xo_gpll0_gpll6_gpll0_sleep_clk, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gp3_clk_src = { + .cmd_rcgr = 0x0a004, + .freq_tbl = ftbl_gp_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp3_clk_src", + .parent_data = gcc_xo_gpll0_gpll6_gpll0_sleep_clk, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_fixed_factor nss_ppe_cdiv_clk_src = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "nss_ppe_cdiv_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap_div nss_ubi0_div_clk_src = { + .reg = 0x68118, + .shift = 0, + .width = 4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "nss_ubi0_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &nss_ubi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ro_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl ftbl_pcie_aux_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), +}; + +static const struct clk_parent_data gcc_xo_gpll0_core_pi_sleep_clk[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .fw_name = "sleep_clk" }, +}; + +static const struct parent_map gcc_xo_gpll0_core_pi_sleep_clk_map[] = { + { P_XO, 0 }, + { P_GPLL0, 2 }, + { P_PI_SLEEP, 6 }, +}; + +static struct clk_rcg2 pcie0_aux_clk_src = { + .cmd_rcgr = 0x75024, + .freq_tbl = ftbl_pcie_aux_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_core_pi_sleep_clk_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcie0_aux_clk_src", + .parent_data = gcc_xo_gpll0_core_pi_sleep_clk, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct clk_parent_data gcc_pcie20_phy0_pipe_clk_xo[] = { + { .fw_name = "pcie20_phy0_pipe_clk" }, + { .fw_name = "xo" }, +}; + +static const struct parent_map gcc_pcie20_phy0_pipe_clk_xo_map[] = { + { P_PCIE20_PHY0_PIPE, 0 }, + { P_XO, 2 }, +}; + +static struct clk_regmap_mux pcie0_pipe_clk_src = { + .reg = 0x7501c, + .shift = 8, + .width = 2, + .parent_map = gcc_pcie20_phy0_pipe_clk_xo_map, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "pcie0_pipe_clk_src", + .parent_data = gcc_pcie20_phy0_pipe_clk_xo, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl ftbl_sdcc_apps_clk_src[] = { + F(144000, P_XO, 16, 12, 125), + F(400000, P_XO, 12, 1, 5), + F(24000000, P_GPLL2, 12, 1, 4), + F(48000000, P_GPLL2, 12, 1, 2), + F(96000000, P_GPLL2, 12, 0, 0), + F(177777778, P_GPLL0, 4.5, 0, 0), + F(192000000, P_GPLL2, 6, 0, 0), + F(384000000, P_GPLL2, 3, 0, 0), + { } +}; + +static const struct clk_parent_data + gcc_xo_gpll0_gpll2_gpll0_out_main_div2[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll2.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_gpll2_gpll0_out_main_div2_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL2, 2 }, + { P_GPLL0_DIV2, 4 }, +}; + +static struct clk_rcg2 sdcc1_apps_clk_src = { + .cmd_rcgr = 0x42004, + .freq_tbl = ftbl_sdcc_apps_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll2_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc1_apps_clk_src", + .parent_data = gcc_xo_gpll0_gpll2_gpll0_out_main_div2, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_usb_aux_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 usb0_aux_clk_src = { + .cmd_rcgr = 0x3e05c, + .freq_tbl = ftbl_usb_aux_clk_src, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_core_pi_sleep_clk_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb0_aux_clk_src", + .parent_data = gcc_xo_gpll0_core_pi_sleep_clk, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_usb_mock_utmi_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(60000000, P_GPLL6, 6, 1, 3), + { } +}; + +static const struct clk_parent_data + gcc_xo_gpll6_gpll0_gpll0_out_main_div2[] = { + { .fw_name = "xo" }, + { .hw = &gpll6.clkr.hw }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, +}; + +static const struct parent_map gcc_xo_gpll6_gpll0_gpll0_out_main_div2_map[] = { + { P_XO, 0 }, + { P_GPLL6, 1 }, + { P_GPLL0, 3 }, + { P_GPLL0_DIV2, 4 }, +}; + +static struct clk_rcg2 usb0_mock_utmi_clk_src = { + .cmd_rcgr = 0x3e020, + .freq_tbl = ftbl_usb_mock_utmi_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll6_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb0_mock_utmi_clk_src", + .parent_data = gcc_xo_gpll6_gpll0_gpll0_out_main_div2, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct clk_parent_data gcc_usb3phy_0_cc_pipe_clk_xo[] = { + { .fw_name = "usb3phy_0_cc_pipe_clk" }, + { .fw_name = "xo" }, +}; + +static const struct parent_map gcc_usb3phy_0_cc_pipe_clk_xo_map[] = { + { P_USB3PHY_0_PIPE, 0 }, + { P_XO, 2 }, +}; + +static struct clk_regmap_mux usb0_pipe_clk_src = { + .reg = 0x3e048, + .shift = 8, + .width = 2, + .parent_map = gcc_usb3phy_0_cc_pipe_clk_xo_map, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "usb0_pipe_clk_src", + .parent_data = gcc_usb3phy_0_cc_pipe_clk_xo, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl ftbl_sdcc_ice_core_clk_src[] = { + F(80000000, P_GPLL0_DIV2, 5, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + F(216000000, P_GPLL6, 5, 0, 0), + F(308570000, P_GPLL6, 3.5, 0, 0), +}; + +static const struct clk_parent_data gcc_xo_gpll0_gpll6_gpll0_div2[] = { + { .fw_name = "xo"}, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll6.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_gpll6_gpll0_div2_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL6, 2 }, + { P_GPLL0_DIV2, 4 }, +}; + +static struct clk_rcg2 sdcc1_ice_core_clk_src = { + .cmd_rcgr = 0x5d000, + .freq_tbl = ftbl_sdcc_ice_core_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll6_gpll0_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc1_ice_core_clk_src", + .parent_data = gcc_xo_gpll0_gpll6_gpll0_div2, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_qdss_stm_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0_DIV2, 8, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + { } +}; + +static struct clk_rcg2 qdss_stm_clk_src = { + .cmd_rcgr = 0x2902C, + .freq_tbl = ftbl_qdss_stm_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "qdss_stm_clk_src", + .parent_data = gcc_xo_gpll0_gpll0_out_main_div2, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_qdss_traceclkin_clk_src[] = { + F(80000000, P_GPLL0_DIV2, 5, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + F(300000000, P_GPLL4, 4, 0, 0), + { } +}; + +static const struct clk_parent_data gcc_xo_gpll4_gpll0_gpll0_div2[] = { + { .fw_name = "xo" }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_main_div2.hw }, +}; + +static const struct parent_map gcc_xo_gpll4_gpll0_gpll0_div2_map[] = { + { P_XO, 0 }, + { P_GPLL4, 1 }, + { P_GPLL0, 2 }, + { P_GPLL0_DIV2, 4 }, +}; + +static struct clk_rcg2 qdss_traceclkin_clk_src = { + .cmd_rcgr = 0x29048, + .freq_tbl = ftbl_qdss_traceclkin_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll4_gpll0_gpll0_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "qdss_traceclkin_clk_src", + .parent_data = gcc_xo_gpll4_gpll0_gpll0_div2, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 usb1_mock_utmi_clk_src = { + .cmd_rcgr = 0x3f020, + .freq_tbl = ftbl_usb_mock_utmi_clk_src, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll6_gpll0_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb1_mock_utmi_clk_src", + .parent_data = gcc_xo_gpll6_gpll0_gpll0_out_main_div2, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_adss_pwm_clk = { + .halt_reg = 0x1c020, + .clkr = { + .enable_reg = 0x1c020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_adss_pwm_clk", + .parent_hws = (const struct clk_hw *[]){ + &adss_pwm_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_apss_ahb_clk = { + .halt_reg = 0x4601c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x0b004, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_apss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &apss_ahb_postdiv_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_system_noc_bfdcd_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0_DIV2, 8, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(133333333, P_GPLL0, 6, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(266666667, P_GPLL0, 3, 0, 0), + { } +}; + +static struct clk_rcg2 system_noc_bfdcd_clk_src = { + .cmd_rcgr = 0x26004, + .freq_tbl = ftbl_system_noc_bfdcd_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll6_gpll0_out_main_div2_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "system_noc_bfdcd_clk_src", + .parent_data = gcc_xo_gpll0_gpll6_gpll0_out_main_div2, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_ubi32_mem_noc_bfdcd_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(307670000, P_BIAS_PLL_NSS_NOC, 1.5, 0, 0), + F(533333333, P_GPLL0, 1.5, 0, 0), + { } +}; + +static const struct clk_parent_data + gcc_xo_gpll0_gpll2_bias_pll_nss_noc_clk[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll2.clkr.hw }, + { .fw_name = "bias_pll_nss_noc_clk" }, +}; + +static const struct parent_map gcc_xo_gpll0_gpll2_bias_pll_nss_noc_clk_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL2, 3 }, + { P_BIAS_PLL_NSS_NOC, 4 }, +}; + +static struct clk_rcg2 ubi32_mem_noc_bfdcd_clk_src = { + .cmd_rcgr = 0x68088, + .freq_tbl = ftbl_ubi32_mem_noc_bfdcd_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll2_bias_pll_nss_noc_clk_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ubi32_mem_noc_bfdcd_clk_src", + .parent_data = gcc_xo_gpll0_gpll2_bias_pll_nss_noc_clk, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_apss_axi_clk = { + .halt_reg = 0x46020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x0b004, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_apss_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &apss_axi_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_ahb_clk = { + .halt_reg = 0x01008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x0b004, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { + .halt_reg = 0x02008, + .clkr = { + .enable_reg = 0x02008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup1_i2c_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { + .halt_reg = 0x02004, + .clkr = { + .enable_reg = 0x02004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup1_spi_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { + .halt_reg = 0x03010, + .clkr = { + .enable_reg = 0x03010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup2_i2c_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { + .halt_reg = 0x0300c, + .clkr = { + .enable_reg = 0x0300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup2_spi_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { + .halt_reg = 0x04010, + .clkr = { + .enable_reg = 0x04010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup3_i2c_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { + .halt_reg = 0x0400c, + .clkr = { + .enable_reg = 0x0400c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup3_spi_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { + .halt_reg = 0x05010, + .clkr = { + .enable_reg = 0x05010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup4_i2c_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { + .halt_reg = 0x0500c, + .clkr = { + .enable_reg = 0x0500c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup4_spi_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = { + .halt_reg = 0x06010, + .clkr = { + .enable_reg = 0x06010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup5_i2c_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup5_i2c_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = { + .halt_reg = 0x0600c, + .clkr = { + .enable_reg = 0x0600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup5_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup5_spi_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = { + .halt_reg = 0x0700c, + .clkr = { + .enable_reg = 0x0700c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup6_spi_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_qup6_spi_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart1_apps_clk = { + .halt_reg = 0x0203c, + .clkr = { + .enable_reg = 0x0203c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart1_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_uart1_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart2_apps_clk = { + .halt_reg = 0x0302c, + .clkr = { + .enable_reg = 0x0302c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart2_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_uart2_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart3_apps_clk = { + .halt_reg = 0x0402c, + .clkr = { + .enable_reg = 0x0402c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart3_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_uart3_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart4_apps_clk = { + .halt_reg = 0x0502c, + .clkr = { + .enable_reg = 0x0502c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart4_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_uart4_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart5_apps_clk = { + .halt_reg = 0x0602c, + .clkr = { + .enable_reg = 0x0602c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart5_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_uart5_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart6_apps_clk = { + .halt_reg = 0x0702c, + .clkr = { + .enable_reg = 0x0702c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart6_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &blsp1_uart6_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_crypto_ahb_clk = { + .halt_reg = 0x16024, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x0b004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_crypto_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_crypto_axi_clk = { + .halt_reg = 0x16020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x0b004, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_crypto_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_crypto_clk = { + .halt_reg = 0x1601c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x0b004, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_crypto_clk", + .parent_hws = (const struct clk_hw *[]){ + &crypto_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_fixed_factor gpll6_out_main_div2 = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "gpll6_out_main_div2", + .parent_hws = (const struct clk_hw *[]){ + &gpll6_main.clkr.hw }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_branch gcc_xo_clk = { + .halt_reg = 0x30030, + .clkr = { + .enable_reg = 0x30030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_xo_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_xo_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x08000, + .clkr = { + .enable_reg = 0x08000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gp1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x09000, + .clkr = { + .enable_reg = 0x09000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gp2_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x0a000, + .clkr = { + .enable_reg = 0x0a000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gp3_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mdio_ahb_clk = { + .halt_reg = 0x58004, + .clkr = { + .enable_reg = 0x58004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mdio_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_crypto_ppe_clk = { + .halt_reg = 0x68310, + .clkr = { + .enable_reg = 0x68310, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_crypto_ppe_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_ce_apb_clk = { + .halt_reg = 0x68174, + .clkr = { + .enable_reg = 0x68174, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_ce_apb_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ce_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_ce_axi_clk = { + .halt_reg = 0x68170, + .clkr = { + .enable_reg = 0x68170, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_ce_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ce_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_cfg_clk = { + .halt_reg = 0x68160, + .clkr = { + .enable_reg = 0x68160, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_cfg_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_crypto_clk = { + .halt_reg = 0x68164, + .clkr = { + .enable_reg = 0x68164, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_crypto_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_crypto_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_csr_clk = { + .halt_reg = 0x68318, + .clkr = { + .enable_reg = 0x68318, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_csr_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ce_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_edma_cfg_clk = { + .halt_reg = 0x6819C, + .clkr = { + .enable_reg = 0x6819C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_edma_cfg_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_edma_clk = { + .halt_reg = 0x68198, + .clkr = { + .enable_reg = 0x68198, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_edma_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_noc_clk = { + .halt_reg = 0x68168, + .clkr = { + .enable_reg = 0x68168, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_noc_clk", + .parent_hws = (const struct clk_hw *[]){ + &snoc_nssnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ubi0_utcm_clk = { + .halt_reg = 0x2606c, + .clkr = { + .enable_reg = 0x2606c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ubi0_utcm_clk", + .parent_hws = (const struct clk_hw *[]){ + &snoc_nssnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_snoc_nssnoc_clk = { + .halt_reg = 0x26070, + .clkr = { + .enable_reg = 0x26070, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_snoc_nssnoc_clk", + .parent_hws = (const struct clk_hw *[]){ + &snoc_nssnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_wcss_ahb_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(133333333, P_GPLL0, 6, 0, 0), + { } +}; + +static const struct freq_tbl ftbl_q6_axi_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(400000000, P_GPLL0, 2, 0, 0), + { } +}; + +static struct clk_rcg2 wcss_ahb_clk_src = { + .cmd_rcgr = 0x59020, + .freq_tbl = ftbl_wcss_ahb_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "wcss_ahb_clk_src", + .parent_data = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct clk_parent_data gcc_xo_gpll0_gpll2_gpll4_gpll6[] = { + { .fw_name = "xo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll2.clkr.hw }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll6.clkr.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_gpll2_gpll4_gpll6_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL2, 2 }, + { P_GPLL4, 3 }, + { P_GPLL6, 4 }, +}; + +static struct clk_rcg2 q6_axi_clk_src = { + .cmd_rcgr = 0x59120, + .freq_tbl = ftbl_q6_axi_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll2_gpll4_gpll6_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "q6_axi_clk_src", + .parent_data = gcc_xo_gpll0_gpll2_gpll4_gpll6, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_lpass_core_axim_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 lpass_core_axim_clk_src = { + .cmd_rcgr = 0x1F020, + .freq_tbl = ftbl_lpass_core_axim_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "lpass_core_axim_clk_src", + .parent_data = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_lpass_snoc_cfg_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(266666667, P_GPLL0, 3, 0, 0), + { } +}; + +static struct clk_rcg2 lpass_snoc_cfg_clk_src = { + .cmd_rcgr = 0x1F040, + .freq_tbl = ftbl_lpass_snoc_cfg_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "lpass_snoc_cfg_clk_src", + .parent_data = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_lpass_q6_axim_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(400000000, P_GPLL0, 2, 0, 0), + { } +}; + +static struct clk_rcg2 lpass_q6_axim_clk_src = { + .cmd_rcgr = 0x1F008, + .freq_tbl = ftbl_lpass_q6_axim_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "lpass_q6_axim_clk_src", + .parent_data = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_rbcpr_wcss_clk_src[] = { + F(24000000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0, 16, 0, 0), + { } +}; + +static struct clk_rcg2 rbcpr_wcss_clk_src = { + .cmd_rcgr = 0x3a00c, + .freq_tbl = ftbl_rbcpr_wcss_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_out_main_div2_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbcpr_wcss_clk_src", + .parent_data = gcc_xo_gpll0_out_main_div2_gpll0, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_lpass_core_axim_clk = { + .halt_reg = 0x1F028, + .clkr = { + .enable_reg = 0x1F028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_core_axim_clk", + .parent_hws = (const struct clk_hw *[]){ + &lpass_core_axim_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_lpass_snoc_cfg_clk = { + .halt_reg = 0x1F048, + .clkr = { + .enable_reg = 0x1F048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_snoc_cfg_clk", + .parent_hws = (const struct clk_hw *[]){ + &lpass_snoc_cfg_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_lpass_q6_axim_clk = { + .halt_reg = 0x1F010, + .clkr = { + .enable_reg = 0x1F010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_q6_axim_clk", + .parent_hws = (const struct clk_hw *[]){ + &lpass_q6_axim_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_lpass_q6_atbm_at_clk = { + .halt_reg = 0x1F018, + .clkr = { + .enable_reg = 0x1F018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_q6_atbm_at_clk", + .parent_hws = (const struct clk_hw *[]){ + &qdss_at_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_lpass_q6_pclkdbg_clk = { + .halt_reg = 0x1F01C, + .clkr = { + .enable_reg = 0x1F01C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_q6_pclkdbg_clk", + .parent_hws = (const struct clk_hw *[]){ + &qdss_dap_sync_clk_src.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_lpass_q6ss_tsctr_1to2_clk = { + .halt_reg = 0x1F014, + .clkr = { + .enable_reg = 0x1F014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_q6ss_tsctr_1to2_clk", + .parent_hws = (const struct clk_hw *[]){ + &qdss_tsctr_div2_clk_src.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_lpass_q6ss_trig_clk = { + .halt_reg = 0x1F038, + .clkr = { + .enable_reg = 0x1F038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_q6ss_trig_clk", + .parent_hws = (const struct clk_hw *[]){ + &qdss_dap_sync_clk_src.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_lpass_tbu_clk = { + .halt_reg = 0x12094, + .clkr = { + .enable_reg = 0xb00c, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_lpass_tbu_clk", + .parent_hws = (const struct clk_hw *[]){ + &lpass_q6_axim_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcnoc_lpass_clk = { + .halt_reg = 0x27020, + .clkr = { + .enable_reg = 0x27020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcnoc_lpass_clk", + .parent_hws = (const struct clk_hw *[]){ + &lpass_core_axim_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mem_noc_lpass_clk = { + .halt_reg = 0x1D044, + .clkr = { + .enable_reg = 0x1D044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mem_noc_lpass_clk", + .parent_hws = (const struct clk_hw *[]){ + &lpass_q6_axim_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_snoc_lpass_cfg_clk = { + .halt_reg = 0x26074, + .clkr = { + .enable_reg = 0x26074, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_snoc_lpass_cfg_clk", + .parent_hws = (const struct clk_hw *[]){ + &lpass_snoc_cfg_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mem_noc_ubi32_clk = { + .halt_reg = 0x1D03C, + .clkr = { + .enable_reg = 0x1D03C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mem_noc_ubi32_clk", + .parent_hws = (const struct clk_hw *[]){ + &ubi32_mem_noc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port1_rx_clk = { + .halt_reg = 0x68240, + .clkr = { + .enable_reg = 0x68240, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port1_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port1_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port1_tx_clk = { + .halt_reg = 0x68244, + .clkr = { + .enable_reg = 0x68244, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port1_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port1_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port2_rx_clk = { + .halt_reg = 0x68248, + .clkr = { + .enable_reg = 0x68248, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port2_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port2_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port2_tx_clk = { + .halt_reg = 0x6824c, + .clkr = { + .enable_reg = 0x6824c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port2_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port2_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port3_rx_clk = { + .halt_reg = 0x68250, + .clkr = { + .enable_reg = 0x68250, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port3_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port3_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port3_tx_clk = { + .halt_reg = 0x68254, + .clkr = { + .enable_reg = 0x68254, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port3_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port3_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port4_rx_clk = { + .halt_reg = 0x68258, + .clkr = { + .enable_reg = 0x68258, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port4_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port4_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port4_tx_clk = { + .halt_reg = 0x6825c, + .clkr = { + .enable_reg = 0x6825c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port4_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port4_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port5_rx_clk = { + .halt_reg = 0x68260, + .clkr = { + .enable_reg = 0x68260, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port5_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_port5_tx_clk = { + .halt_reg = 0x68264, + .clkr = { + .enable_reg = 0x68264, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_port5_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_ppe_cfg_clk = { + .halt_reg = 0x68194, + .clkr = { + .enable_reg = 0x68194, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_ppe_cfg_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_ppe_clk = { + .halt_reg = 0x68190, + .clkr = { + .enable_reg = 0x68190, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_ppe_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_ppe_ipe_clk = { + .halt_reg = 0x68338, + .clkr = { + .enable_reg = 0x68338, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_ppe_ipe_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nss_ptp_ref_clk = { + .halt_reg = 0x6816C, + .clkr = { + .enable_reg = 0x6816C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nss_ptp_ref_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_cdiv_clk_src.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_ce_apb_clk = { + .halt_reg = 0x6830C, + .clkr = { + .enable_reg = 0x6830C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_ce_apb_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ce_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_ce_axi_clk = { + .halt_reg = 0x68308, + .clkr = { + .enable_reg = 0x68308, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_ce_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ce_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_crypto_clk = { + .halt_reg = 0x68314, + .clkr = { + .enable_reg = 0x68314, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_crypto_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_crypto_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_ppe_cfg_clk = { + .halt_reg = 0x68304, + .clkr = { + .enable_reg = 0x68304, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_ppe_cfg_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_ppe_clk = { + .halt_reg = 0x68300, + .clkr = { + .enable_reg = 0x68300, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_ppe_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_qosgen_ref_clk = { + .halt_reg = 0x68180, + .clkr = { + .enable_reg = 0x68180, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_qosgen_ref_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_xo_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_snoc_clk = { + .halt_reg = 0x68188, + .clkr = { + .enable_reg = 0x68188, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_snoc_clk", + .parent_hws = (const struct clk_hw *[]){ + &system_noc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_timeout_ref_clk = { + .halt_reg = 0x68184, + .clkr = { + .enable_reg = 0x68184, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_timeout_ref_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_xo_div4_clk_src.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_nssnoc_ubi0_ahb_clk = { + .halt_reg = 0x68270, + .clkr = { + .enable_reg = 0x68270, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_nssnoc_ubi0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ce_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_port1_mac_clk = { + .halt_reg = 0x68320, + .clkr = { + .enable_reg = 0x68320, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_port1_mac_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_port2_mac_clk = { + .halt_reg = 0x68324, + .clkr = { + .enable_reg = 0x68324, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_port2_mac_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_port3_mac_clk = { + .halt_reg = 0x68328, + .clkr = { + .enable_reg = 0x68328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_port3_mac_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_port4_mac_clk = { + .halt_reg = 0x6832c, + .clkr = { + .enable_reg = 0x6832c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_port4_mac_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_port5_mac_clk = { + .halt_reg = 0x68330, + .clkr = { + .enable_reg = 0x68330, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_port5_mac_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ppe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ubi0_ahb_clk = { + .halt_reg = 0x6820C, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x6820C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ubi0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ce_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ubi0_axi_clk = { + .halt_reg = 0x68200, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x68200, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ubi0_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &ubi32_mem_noc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ubi0_nc_axi_clk = { + .halt_reg = 0x68204, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x68204, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ubi0_nc_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &snoc_nssnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ubi0_core_clk = { + .halt_reg = 0x68210, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x68210, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ubi0_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_ubi0_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_ahb_clk = { + .halt_reg = 0x75010, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_aux_clk = { + .halt_reg = 0x75014, + .clkr = { + .enable_reg = 0x75014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcie0_aux_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_axi_m_clk = { + .halt_reg = 0x75008, + .clkr = { + .enable_reg = 0x75008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_axi_m_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcie0_axi_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_axi_s_clk = { + .halt_reg = 0x7500c, + .clkr = { + .enable_reg = 0x7500c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_axi_s_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcie0_axi_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_pcie0_axi_clk = { + .halt_reg = 0x26048, + .clkr = { + .enable_reg = 0x26048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_pcie0_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcie0_axi_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_pipe_clk = { + .halt_reg = 0x75018, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x75018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_pipe_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcie0_pipe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_prng_ahb_clk = { + .halt_reg = 0x13004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x0b004, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_prng_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qdss_dap_clk = { + .halt_reg = 0x29084, + .clkr = { + .enable_reg = 0x29084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qdss_dap_clk", + .parent_hws = (const struct clk_hw *[]){ + &qdss_dap_sync_clk_src.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qpic_ahb_clk = { + .halt_reg = 0x57024, + .clkr = { + .enable_reg = 0x57024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qpic_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qpic_clk = { + .halt_reg = 0x57020, + .clkr = { + .enable_reg = 0x57020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qpic_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x4201c, + .clkr = { + .enable_reg = 0x4201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x42018, + .clkr = { + .enable_reg = 0x42018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &sdcc1_apps_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_ahb_clk = { + .halt_reg = 0x56008, + .clkr = { + .enable_reg = 0x56008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port1_rx_clk = { + .halt_reg = 0x56010, + .clkr = { + .enable_reg = 0x56010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port1_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port1_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port1_tx_clk = { + .halt_reg = 0x56014, + .clkr = { + .enable_reg = 0x56014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port1_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port1_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port2_rx_clk = { + .halt_reg = 0x56018, + .clkr = { + .enable_reg = 0x56018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port2_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port2_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port2_tx_clk = { + .halt_reg = 0x5601c, + .clkr = { + .enable_reg = 0x5601c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port2_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port2_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port3_rx_clk = { + .halt_reg = 0x56020, + .clkr = { + .enable_reg = 0x56020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port3_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port3_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port3_tx_clk = { + .halt_reg = 0x56024, + .clkr = { + .enable_reg = 0x56024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port3_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port3_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port4_rx_clk = { + .halt_reg = 0x56028, + .clkr = { + .enable_reg = 0x56028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port4_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port4_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port4_tx_clk = { + .halt_reg = 0x5602c, + .clkr = { + .enable_reg = 0x5602c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port4_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port4_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port5_rx_clk = { + .halt_reg = 0x56030, + .clkr = { + .enable_reg = 0x56030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port5_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_port5_tx_clk = { + .halt_reg = 0x56034, + .clkr = { + .enable_reg = 0x56034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_port5_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy0_sys_clk = { + .halt_reg = 0x5600C, + .clkr = { + .enable_reg = 0x5600C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy0_sys_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_xo_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy1_ahb_clk = { + .halt_reg = 0x56108, + .clkr = { + .enable_reg = 0x56108, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy1_port5_rx_clk = { + .halt_reg = 0x56110, + .clkr = { + .enable_reg = 0x56110, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy1_port5_rx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_rx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy1_port5_tx_clk = { + .halt_reg = 0x56114, + .clkr = { + .enable_reg = 0x56114, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy1_port5_tx_clk", + .parent_hws = (const struct clk_hw *[]){ + &nss_port5_tx_div_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_uniphy1_sys_clk = { + .halt_reg = 0x5610C, + .clkr = { + .enable_reg = 0x5610C, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_uniphy1_sys_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_xo_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb0_aux_clk = { + .halt_reg = 0x3e044, + .clkr = { + .enable_reg = 0x3e044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb0_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &usb0_aux_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb0_master_clk = { + .halt_reg = 0x3e000, + .clkr = { + .enable_reg = 0x3e000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb0_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &usb0_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_snoc_bus_timeout2_ahb_clk = { + .halt_reg = 0x47014, + .clkr = { + .enable_reg = 0x47014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_snoc_bus_timeout2_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &usb0_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_rcg2 pcie0_rchng_clk_src = { + .cmd_rcgr = 0x75070, + .freq_tbl = ftbl_pcie_rchng_clk_src, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcie0_rchng_clk_src", + .parent_data = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_pcie0_rchng_clk = { + .halt_reg = 0x75070, + .clkr = { + .enable_reg = 0x75070, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_rchng_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcie0_rchng_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_axi_s_bridge_clk = { + .halt_reg = 0x75048, + .clkr = { + .enable_reg = 0x75048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_axi_s_bridge_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcie0_axi_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_usb0_axi_clk = { + .halt_reg = 0x26040, + .clkr = { + .enable_reg = 0x26040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_usb0_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &usb0_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb0_mock_utmi_clk = { + .halt_reg = 0x3e008, + .clkr = { + .enable_reg = 0x3e008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb0_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &usb0_mock_utmi_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb0_phy_cfg_ahb_clk = { + .halt_reg = 0x3e080, + .clkr = { + .enable_reg = 0x3e080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb0_phy_cfg_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb0_pipe_clk = { + .halt_reg = 0x3e040, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x3e040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb0_pipe_clk", + .parent_hws = (const struct clk_hw *[]){ + &usb0_pipe_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb0_sleep_clk = { + .halt_reg = 0x3e004, + .clkr = { + .enable_reg = 0x3e004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb0_sleep_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_sleep_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb1_master_clk = { + .halt_reg = 0x3f000, + .clkr = { + .enable_reg = 0x3f000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb1_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb1_mock_utmi_clk = { + .halt_reg = 0x3f008, + .clkr = { + .enable_reg = 0x3f008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb1_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &usb1_mock_utmi_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb1_phy_cfg_ahb_clk = { + .halt_reg = 0x3f080, + .clkr = { + .enable_reg = 0x3f080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb1_phy_cfg_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb1_sleep_clk = { + .halt_reg = 0x3f004, + .clkr = { + .enable_reg = 0x3f004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb1_sleep_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_sleep_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cmn_12gpll_ahb_clk = { + .halt_reg = 0x56308, + .clkr = { + .enable_reg = 0x56308, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cmn_12gpll_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cmn_12gpll_sys_clk = { + .halt_reg = 0x5630c, + .clkr = { + .enable_reg = 0x5630c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cmn_12gpll_sys_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_xo_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ice_core_clk = { + .halt_reg = 0x5d014, + .clkr = { + .enable_reg = 0x5d014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &sdcc1_ice_core_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_dcc_clk = { + .halt_reg = 0x77004, + .clkr = { + .enable_reg = 0x77004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_dcc_clk", + .parent_hws = (const struct clk_hw *[]){ + &pcnoc_bfdcd_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct alpha_pll_config ubi32_pll_config = { + .l = 0x3e, + .alpha = 0x57, + .config_ctl_val = 0x240d6aa8, + .config_ctl_hi_val = 0x3c2, + .main_output_mask = BIT(0), + .aux_output_mask = BIT(1), + .pre_div_val = 0x0, + .pre_div_mask = BIT(12), + .post_div_val = 0x0, + .post_div_mask = GENMASK(9, 8), +}; + +static const struct alpha_pll_config nss_crypto_pll_config = { + .l = 0x32, + .alpha = 0x0, + .alpha_hi = 0x0, + .config_ctl_val = 0x4001055b, + .main_output_mask = BIT(0), + .pre_div_val = 0x0, + .pre_div_mask = GENMASK(14, 12), + .post_div_val = 0x1 << 8, + .post_div_mask = GENMASK(11, 8), + .vco_mask = GENMASK(21, 20), + .vco_val = 0x0, + .alpha_en_mask = BIT(24), +}; + +static struct clk_hw *gcc_ipq6018_hws[] = { + &gpll0_out_main_div2.hw, + &gcc_xo_div4_clk_src.hw, + &nss_ppe_cdiv_clk_src.hw, + &gpll6_out_main_div2.hw, + &qdss_dap_sync_clk_src.hw, + &qdss_tsctr_div2_clk_src.hw, +}; + +static struct clk_regmap *gcc_ipq6018_clks[] = { + [GPLL0_MAIN] = &gpll0_main.clkr, + [GPLL0] = &gpll0.clkr, + [UBI32_PLL_MAIN] = &ubi32_pll_main.clkr, + [UBI32_PLL] = &ubi32_pll.clkr, + [GPLL6_MAIN] = &gpll6_main.clkr, + [GPLL6] = &gpll6.clkr, + [GPLL4_MAIN] = &gpll4_main.clkr, + [GPLL4] = &gpll4.clkr, + [PCNOC_BFDCD_CLK_SRC] = &pcnoc_bfdcd_clk_src.clkr, + [GPLL2_MAIN] = &gpll2_main.clkr, + [GPLL2] = &gpll2.clkr, + [NSS_CRYPTO_PLL_MAIN] = &nss_crypto_pll_main.clkr, + [NSS_CRYPTO_PLL] = &nss_crypto_pll.clkr, + [QDSS_TSCTR_CLK_SRC] = &qdss_tsctr_clk_src.clkr, + [QDSS_AT_CLK_SRC] = &qdss_at_clk_src.clkr, + [NSS_PPE_CLK_SRC] = &nss_ppe_clk_src.clkr, + [GCC_XO_CLK_SRC] = &gcc_xo_clk_src.clkr, + [SYSTEM_NOC_BFDCD_CLK_SRC] = &system_noc_bfdcd_clk_src.clkr, + [SNOC_NSSNOC_BFDCD_CLK_SRC] = &snoc_nssnoc_bfdcd_clk_src.clkr, + [NSS_CE_CLK_SRC] = &nss_ce_clk_src.clkr, + [GCC_SLEEP_CLK_SRC] = &gcc_sleep_clk_src.clkr, + [APSS_AHB_CLK_SRC] = &apss_ahb_clk_src.clkr, + [NSS_PORT5_RX_CLK_SRC] = &nss_port5_rx_clk_src.clkr, + [NSS_PORT5_TX_CLK_SRC] = &nss_port5_tx_clk_src.clkr, + [UBI32_MEM_NOC_BFDCD_CLK_SRC] = &ubi32_mem_noc_bfdcd_clk_src.clkr, + [PCIE0_AXI_CLK_SRC] = &pcie0_axi_clk_src.clkr, + [USB0_MASTER_CLK_SRC] = &usb0_master_clk_src.clkr, + [APSS_AHB_POSTDIV_CLK_SRC] = &apss_ahb_postdiv_clk_src.clkr, + [NSS_PORT1_RX_CLK_SRC] = &nss_port1_rx_clk_src.clkr, + [NSS_PORT1_TX_CLK_SRC] = &nss_port1_tx_clk_src.clkr, + [NSS_PORT2_RX_CLK_SRC] = &nss_port2_rx_clk_src.clkr, + [NSS_PORT2_TX_CLK_SRC] = &nss_port2_tx_clk_src.clkr, + [NSS_PORT3_RX_CLK_SRC] = &nss_port3_rx_clk_src.clkr, + [NSS_PORT3_TX_CLK_SRC] = &nss_port3_tx_clk_src.clkr, + [NSS_PORT4_RX_CLK_SRC] = &nss_port4_rx_clk_src.clkr, + [NSS_PORT4_TX_CLK_SRC] = &nss_port4_tx_clk_src.clkr, + [NSS_PORT5_RX_DIV_CLK_SRC] = &nss_port5_rx_div_clk_src.clkr, + [NSS_PORT5_TX_DIV_CLK_SRC] = &nss_port5_tx_div_clk_src.clkr, + [APSS_AXI_CLK_SRC] = &apss_axi_clk_src.clkr, + [NSS_CRYPTO_CLK_SRC] = &nss_crypto_clk_src.clkr, + [NSS_PORT1_RX_DIV_CLK_SRC] = &nss_port1_rx_div_clk_src.clkr, + [NSS_PORT1_TX_DIV_CLK_SRC] = &nss_port1_tx_div_clk_src.clkr, + [NSS_PORT2_RX_DIV_CLK_SRC] = &nss_port2_rx_div_clk_src.clkr, + [NSS_PORT2_TX_DIV_CLK_SRC] = &nss_port2_tx_div_clk_src.clkr, + [NSS_PORT3_RX_DIV_CLK_SRC] = &nss_port3_rx_div_clk_src.clkr, + [NSS_PORT3_TX_DIV_CLK_SRC] = &nss_port3_tx_div_clk_src.clkr, + [NSS_PORT4_RX_DIV_CLK_SRC] = &nss_port4_rx_div_clk_src.clkr, + [NSS_PORT4_TX_DIV_CLK_SRC] = &nss_port4_tx_div_clk_src.clkr, + [NSS_UBI0_CLK_SRC] = &nss_ubi0_clk_src.clkr, + [ADSS_PWM_CLK_SRC] = &adss_pwm_clk_src.clkr, + [BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr, + [BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr, + [BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr, + [BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr, + [BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr, + [BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr, + [BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr, + [BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr, + [BLSP1_QUP5_I2C_APPS_CLK_SRC] = &blsp1_qup5_i2c_apps_clk_src.clkr, + [BLSP1_QUP5_SPI_APPS_CLK_SRC] = &blsp1_qup5_spi_apps_clk_src.clkr, + [BLSP1_QUP6_I2C_APPS_CLK_SRC] = &blsp1_qup6_i2c_apps_clk_src.clkr, + [BLSP1_QUP6_SPI_APPS_CLK_SRC] = &blsp1_qup6_spi_apps_clk_src.clkr, + [BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr, + [BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr, + [BLSP1_UART3_APPS_CLK_SRC] = &blsp1_uart3_apps_clk_src.clkr, + [BLSP1_UART4_APPS_CLK_SRC] = &blsp1_uart4_apps_clk_src.clkr, + [BLSP1_UART5_APPS_CLK_SRC] = &blsp1_uart5_apps_clk_src.clkr, + [BLSP1_UART6_APPS_CLK_SRC] = &blsp1_uart6_apps_clk_src.clkr, + [CRYPTO_CLK_SRC] = &crypto_clk_src.clkr, + [GP1_CLK_SRC] = &gp1_clk_src.clkr, + [GP2_CLK_SRC] = &gp2_clk_src.clkr, + [GP3_CLK_SRC] = &gp3_clk_src.clkr, + [NSS_UBI0_DIV_CLK_SRC] = &nss_ubi0_div_clk_src.clkr, + [PCIE0_AUX_CLK_SRC] = &pcie0_aux_clk_src.clkr, + [PCIE0_PIPE_CLK_SRC] = &pcie0_pipe_clk_src.clkr, + [SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr, + [USB0_AUX_CLK_SRC] = &usb0_aux_clk_src.clkr, + [USB0_MOCK_UTMI_CLK_SRC] = &usb0_mock_utmi_clk_src.clkr, + [USB0_PIPE_CLK_SRC] = &usb0_pipe_clk_src.clkr, + [USB1_MOCK_UTMI_CLK_SRC] = &usb1_mock_utmi_clk_src.clkr, + [GCC_ADSS_PWM_CLK] = &gcc_adss_pwm_clk.clkr, + [GCC_APSS_AHB_CLK] = &gcc_apss_ahb_clk.clkr, + [GCC_APSS_AXI_CLK] = &gcc_apss_axi_clk.clkr, + [GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr, + [GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr, + [GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr, + [GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr, + [GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr, + [GCC_BLSP1_QUP5_I2C_APPS_CLK] = &gcc_blsp1_qup5_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP5_SPI_APPS_CLK] = &gcc_blsp1_qup5_spi_apps_clk.clkr, + [GCC_BLSP1_QUP6_SPI_APPS_CLK] = &gcc_blsp1_qup6_spi_apps_clk.clkr, + [GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr, + [GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr, + [GCC_BLSP1_UART3_APPS_CLK] = &gcc_blsp1_uart3_apps_clk.clkr, + [GCC_BLSP1_UART4_APPS_CLK] = &gcc_blsp1_uart4_apps_clk.clkr, + [GCC_BLSP1_UART5_APPS_CLK] = &gcc_blsp1_uart5_apps_clk.clkr, + [GCC_BLSP1_UART6_APPS_CLK] = &gcc_blsp1_uart6_apps_clk.clkr, + [GCC_CRYPTO_AHB_CLK] = &gcc_crypto_ahb_clk.clkr, + [GCC_CRYPTO_AXI_CLK] = &gcc_crypto_axi_clk.clkr, + [GCC_CRYPTO_CLK] = &gcc_crypto_clk.clkr, + [GCC_XO_CLK] = &gcc_xo_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_MDIO_AHB_CLK] = &gcc_mdio_ahb_clk.clkr, + [GCC_CRYPTO_PPE_CLK] = &gcc_crypto_ppe_clk.clkr, + [GCC_NSS_CE_APB_CLK] = &gcc_nss_ce_apb_clk.clkr, + [GCC_NSS_CE_AXI_CLK] = &gcc_nss_ce_axi_clk.clkr, + [GCC_NSS_CFG_CLK] = &gcc_nss_cfg_clk.clkr, + [GCC_NSS_CRYPTO_CLK] = &gcc_nss_crypto_clk.clkr, + [GCC_NSS_CSR_CLK] = &gcc_nss_csr_clk.clkr, + [GCC_NSS_EDMA_CFG_CLK] = &gcc_nss_edma_cfg_clk.clkr, + [GCC_NSS_EDMA_CLK] = &gcc_nss_edma_clk.clkr, + [GCC_NSS_NOC_CLK] = &gcc_nss_noc_clk.clkr, + [GCC_UBI0_UTCM_CLK] = &gcc_ubi0_utcm_clk.clkr, + [GCC_SNOC_NSSNOC_CLK] = &gcc_snoc_nssnoc_clk.clkr, + [GCC_NSS_PORT1_RX_CLK] = &gcc_nss_port1_rx_clk.clkr, + [GCC_NSS_PORT1_TX_CLK] = &gcc_nss_port1_tx_clk.clkr, + [GCC_NSS_PORT2_RX_CLK] = &gcc_nss_port2_rx_clk.clkr, + [GCC_NSS_PORT2_TX_CLK] = &gcc_nss_port2_tx_clk.clkr, + [GCC_NSS_PORT3_RX_CLK] = &gcc_nss_port3_rx_clk.clkr, + [GCC_NSS_PORT3_TX_CLK] = &gcc_nss_port3_tx_clk.clkr, + [GCC_NSS_PORT4_RX_CLK] = &gcc_nss_port4_rx_clk.clkr, + [GCC_NSS_PORT4_TX_CLK] = &gcc_nss_port4_tx_clk.clkr, + [GCC_NSS_PORT5_RX_CLK] = &gcc_nss_port5_rx_clk.clkr, + [GCC_NSS_PORT5_TX_CLK] = &gcc_nss_port5_tx_clk.clkr, + [GCC_NSS_PPE_CFG_CLK] = &gcc_nss_ppe_cfg_clk.clkr, + [GCC_NSS_PPE_CLK] = &gcc_nss_ppe_clk.clkr, + [GCC_NSS_PPE_IPE_CLK] = &gcc_nss_ppe_ipe_clk.clkr, + [GCC_NSS_PTP_REF_CLK] = &gcc_nss_ptp_ref_clk.clkr, + [GCC_NSSNOC_CE_APB_CLK] = &gcc_nssnoc_ce_apb_clk.clkr, + [GCC_NSSNOC_CE_AXI_CLK] = &gcc_nssnoc_ce_axi_clk.clkr, + [GCC_NSSNOC_CRYPTO_CLK] = &gcc_nssnoc_crypto_clk.clkr, + [GCC_NSSNOC_PPE_CFG_CLK] = &gcc_nssnoc_ppe_cfg_clk.clkr, + [GCC_NSSNOC_PPE_CLK] = &gcc_nssnoc_ppe_clk.clkr, + [GCC_NSSNOC_QOSGEN_REF_CLK] = &gcc_nssnoc_qosgen_ref_clk.clkr, + [GCC_NSSNOC_SNOC_CLK] = &gcc_nssnoc_snoc_clk.clkr, + [GCC_NSSNOC_TIMEOUT_REF_CLK] = &gcc_nssnoc_timeout_ref_clk.clkr, + [GCC_NSSNOC_UBI0_AHB_CLK] = &gcc_nssnoc_ubi0_ahb_clk.clkr, + [GCC_PORT1_MAC_CLK] = &gcc_port1_mac_clk.clkr, + [GCC_PORT2_MAC_CLK] = &gcc_port2_mac_clk.clkr, + [GCC_PORT3_MAC_CLK] = &gcc_port3_mac_clk.clkr, + [GCC_PORT4_MAC_CLK] = &gcc_port4_mac_clk.clkr, + [GCC_PORT5_MAC_CLK] = &gcc_port5_mac_clk.clkr, + [GCC_UBI0_AHB_CLK] = &gcc_ubi0_ahb_clk.clkr, + [GCC_UBI0_AXI_CLK] = &gcc_ubi0_axi_clk.clkr, + [GCC_UBI0_NC_AXI_CLK] = &gcc_ubi0_nc_axi_clk.clkr, + [GCC_UBI0_CORE_CLK] = &gcc_ubi0_core_clk.clkr, + [GCC_PCIE0_AHB_CLK] = &gcc_pcie0_ahb_clk.clkr, + [GCC_PCIE0_AUX_CLK] = &gcc_pcie0_aux_clk.clkr, + [GCC_PCIE0_AXI_M_CLK] = &gcc_pcie0_axi_m_clk.clkr, + [GCC_PCIE0_AXI_S_CLK] = &gcc_pcie0_axi_s_clk.clkr, + [GCC_SYS_NOC_PCIE0_AXI_CLK] = &gcc_sys_noc_pcie0_axi_clk.clkr, + [GCC_PCIE0_PIPE_CLK] = &gcc_pcie0_pipe_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_QDSS_DAP_CLK] = &gcc_qdss_dap_clk.clkr, + [GCC_QPIC_AHB_CLK] = &gcc_qpic_ahb_clk.clkr, + [GCC_QPIC_CLK] = &gcc_qpic_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_UNIPHY0_AHB_CLK] = &gcc_uniphy0_ahb_clk.clkr, + [GCC_UNIPHY0_PORT1_RX_CLK] = &gcc_uniphy0_port1_rx_clk.clkr, + [GCC_UNIPHY0_PORT1_TX_CLK] = &gcc_uniphy0_port1_tx_clk.clkr, + [GCC_UNIPHY0_PORT2_RX_CLK] = &gcc_uniphy0_port2_rx_clk.clkr, + [GCC_UNIPHY0_PORT2_TX_CLK] = &gcc_uniphy0_port2_tx_clk.clkr, + [GCC_UNIPHY0_PORT3_RX_CLK] = &gcc_uniphy0_port3_rx_clk.clkr, + [GCC_UNIPHY0_PORT3_TX_CLK] = &gcc_uniphy0_port3_tx_clk.clkr, + [GCC_UNIPHY0_PORT4_RX_CLK] = &gcc_uniphy0_port4_rx_clk.clkr, + [GCC_UNIPHY0_PORT4_TX_CLK] = &gcc_uniphy0_port4_tx_clk.clkr, + [GCC_UNIPHY0_PORT5_RX_CLK] = &gcc_uniphy0_port5_rx_clk.clkr, + [GCC_UNIPHY0_PORT5_TX_CLK] = &gcc_uniphy0_port5_tx_clk.clkr, + [GCC_UNIPHY0_SYS_CLK] = &gcc_uniphy0_sys_clk.clkr, + [GCC_UNIPHY1_AHB_CLK] = &gcc_uniphy1_ahb_clk.clkr, + [GCC_UNIPHY1_PORT5_RX_CLK] = &gcc_uniphy1_port5_rx_clk.clkr, + [GCC_UNIPHY1_PORT5_TX_CLK] = &gcc_uniphy1_port5_tx_clk.clkr, + [GCC_UNIPHY1_SYS_CLK] = &gcc_uniphy1_sys_clk.clkr, + [GCC_USB0_AUX_CLK] = &gcc_usb0_aux_clk.clkr, + [GCC_SYS_NOC_USB0_AXI_CLK] = &gcc_sys_noc_usb0_axi_clk.clkr, + [GCC_SNOC_BUS_TIMEOUT2_AHB_CLK] = &gcc_snoc_bus_timeout2_ahb_clk.clkr, + [GCC_USB0_MASTER_CLK] = &gcc_usb0_master_clk.clkr, + [GCC_USB0_MOCK_UTMI_CLK] = &gcc_usb0_mock_utmi_clk.clkr, + [GCC_USB0_PHY_CFG_AHB_CLK] = &gcc_usb0_phy_cfg_ahb_clk.clkr, + [GCC_USB0_PIPE_CLK] = &gcc_usb0_pipe_clk.clkr, + [GCC_USB0_SLEEP_CLK] = &gcc_usb0_sleep_clk.clkr, + [GCC_USB1_MASTER_CLK] = &gcc_usb1_master_clk.clkr, + [GCC_USB1_MOCK_UTMI_CLK] = &gcc_usb1_mock_utmi_clk.clkr, + [GCC_USB1_PHY_CFG_AHB_CLK] = &gcc_usb1_phy_cfg_ahb_clk.clkr, + [GCC_USB1_SLEEP_CLK] = &gcc_usb1_sleep_clk.clkr, + [GCC_CMN_12GPLL_AHB_CLK] = &gcc_cmn_12gpll_ahb_clk.clkr, + [GCC_CMN_12GPLL_SYS_CLK] = &gcc_cmn_12gpll_sys_clk.clkr, + [GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr, + [SDCC1_ICE_CORE_CLK_SRC] = &sdcc1_ice_core_clk_src.clkr, + [GCC_DCC_CLK] = &gcc_dcc_clk.clkr, + [PCIE0_RCHNG_CLK_SRC] = &pcie0_rchng_clk_src.clkr, + [GCC_PCIE0_AXI_S_BRIDGE_CLK] = &gcc_pcie0_axi_s_bridge_clk.clkr, + [PCIE0_RCHNG_CLK] = &gcc_pcie0_rchng_clk.clkr, + [WCSS_AHB_CLK_SRC] = &wcss_ahb_clk_src.clkr, + [Q6_AXI_CLK_SRC] = &q6_axi_clk_src.clkr, + [RBCPR_WCSS_CLK_SRC] = &rbcpr_wcss_clk_src.clkr, + [GCC_LPASS_CORE_AXIM_CLK] = &gcc_lpass_core_axim_clk.clkr, + [LPASS_CORE_AXIM_CLK_SRC] = &lpass_core_axim_clk_src.clkr, + [GCC_LPASS_SNOC_CFG_CLK] = &gcc_lpass_snoc_cfg_clk.clkr, + [LPASS_SNOC_CFG_CLK_SRC] = &lpass_snoc_cfg_clk_src.clkr, + [GCC_LPASS_Q6_AXIM_CLK] = &gcc_lpass_q6_axim_clk.clkr, + [LPASS_Q6_AXIM_CLK_SRC] = &lpass_q6_axim_clk_src.clkr, + [GCC_LPASS_Q6_ATBM_AT_CLK] = &gcc_lpass_q6_atbm_at_clk.clkr, + [GCC_LPASS_Q6_PCLKDBG_CLK] = &gcc_lpass_q6_pclkdbg_clk.clkr, + [GCC_LPASS_Q6SS_TSCTR_1TO2_CLK] = &gcc_lpass_q6ss_tsctr_1to2_clk.clkr, + [GCC_LPASS_Q6SS_TRIG_CLK] = &gcc_lpass_q6ss_trig_clk.clkr, + [GCC_LPASS_TBU_CLK] = &gcc_lpass_tbu_clk.clkr, + [GCC_PCNOC_LPASS_CLK] = &gcc_pcnoc_lpass_clk.clkr, + [GCC_MEM_NOC_UBI32_CLK] = &gcc_mem_noc_ubi32_clk.clkr, + [GCC_MEM_NOC_LPASS_CLK] = &gcc_mem_noc_lpass_clk.clkr, + [GCC_SNOC_LPASS_CFG_CLK] = &gcc_snoc_lpass_cfg_clk.clkr, + [QDSS_STM_CLK_SRC] = &qdss_stm_clk_src.clkr, + [QDSS_TRACECLKIN_CLK_SRC] = &qdss_traceclkin_clk_src.clkr, +}; + +static const struct qcom_reset_map gcc_ipq6018_resets[] = { + [GCC_BLSP1_BCR] = { 0x01000, 0 }, + [GCC_BLSP1_QUP1_BCR] = { 0x02000, 0 }, + [GCC_BLSP1_UART1_BCR] = { 0x02038, 0 }, + [GCC_BLSP1_QUP2_BCR] = { 0x03008, 0 }, + [GCC_BLSP1_UART2_BCR] = { 0x03028, 0 }, + [GCC_BLSP1_QUP3_BCR] = { 0x04008, 0 }, + [GCC_BLSP1_UART3_BCR] = { 0x04028, 0 }, + [GCC_BLSP1_QUP4_BCR] = { 0x05008, 0 }, + [GCC_BLSP1_UART4_BCR] = { 0x05028, 0 }, + [GCC_BLSP1_QUP5_BCR] = { 0x06008, 0 }, + [GCC_BLSP1_UART5_BCR] = { 0x06028, 0 }, + [GCC_BLSP1_QUP6_BCR] = { 0x07008, 0 }, + [GCC_BLSP1_UART6_BCR] = { 0x07028, 0 }, + [GCC_IMEM_BCR] = { 0x0e000, 0 }, + [GCC_SMMU_BCR] = { 0x12000, 0 }, + [GCC_APSS_TCU_BCR] = { 0x12050, 0 }, + [GCC_SMMU_XPU_BCR] = { 0x12054, 0 }, + [GCC_PCNOC_TBU_BCR] = { 0x12058, 0 }, + [GCC_SMMU_CFG_BCR] = { 0x1208c, 0 }, + [GCC_PRNG_BCR] = { 0x13000, 0 }, + [GCC_BOOT_ROM_BCR] = { 0x13008, 0 }, + [GCC_CRYPTO_BCR] = { 0x16000, 0 }, + [GCC_WCSS_BCR] = { 0x18000, 0 }, + [GCC_WCSS_Q6_BCR] = { 0x18100, 0 }, + [GCC_NSS_BCR] = { 0x19000, 0 }, + [GCC_SEC_CTRL_BCR] = { 0x1a000, 0 }, + [GCC_ADSS_BCR] = { 0x1c000, 0 }, + [GCC_DDRSS_BCR] = { 0x1e000, 0 }, + [GCC_SYSTEM_NOC_BCR] = { 0x26000, 0 }, + [GCC_PCNOC_BCR] = { 0x27018, 0 }, + [GCC_TCSR_BCR] = { 0x28000, 0 }, + [GCC_QDSS_BCR] = { 0x29000, 0 }, + [GCC_DCD_BCR] = { 0x2a000, 0 }, + [GCC_MSG_RAM_BCR] = { 0x2b000, 0 }, + [GCC_MPM_BCR] = { 0x2c000, 0 }, + [GCC_SPDM_BCR] = { 0x2f000, 0 }, + [GCC_RBCPR_BCR] = { 0x33000, 0 }, + [GCC_RBCPR_MX_BCR] = { 0x33014, 0 }, + [GCC_TLMM_BCR] = { 0x34000, 0 }, + [GCC_RBCPR_WCSS_BCR] = { 0x3a000, 0 }, + [GCC_USB0_PHY_BCR] = { 0x3e034, 0 }, + [GCC_USB3PHY_0_PHY_BCR] = { 0x3e03c, 0 }, + [GCC_USB0_BCR] = { 0x3e070, 0 }, + [GCC_USB1_BCR] = { 0x3f070, 0 }, + [GCC_QUSB2_0_PHY_BCR] = { 0x4103c, 0 }, + [GCC_QUSB2_1_PHY_BCR] = { 0x41040, 0 }, + [GCC_SDCC1_BCR] = { 0x42000, 0 }, + [GCC_SNOC_BUS_TIMEOUT0_BCR] = { 0x47000, 0 }, + [GCC_SNOC_BUS_TIMEOUT1_BCR] = { 0x47008, 0 }, + [GCC_SNOC_BUS_TIMEOUT2_BCR] = { 0x47010, 0 }, + [GCC_PCNOC_BUS_TIMEOUT0_BCR] = { 0x48000, 0 }, + [GCC_PCNOC_BUS_TIMEOUT1_BCR] = { 0x48008, 0 }, + [GCC_PCNOC_BUS_TIMEOUT2_BCR] = { 0x48010, 0 }, + [GCC_PCNOC_BUS_TIMEOUT3_BCR] = { 0x48018, 0 }, + [GCC_PCNOC_BUS_TIMEOUT4_BCR] = { 0x48020, 0 }, + [GCC_PCNOC_BUS_TIMEOUT5_BCR] = { 0x48028, 0 }, + [GCC_PCNOC_BUS_TIMEOUT6_BCR] = { 0x48030, 0 }, + [GCC_PCNOC_BUS_TIMEOUT7_BCR] = { 0x48038, 0 }, + [GCC_PCNOC_BUS_TIMEOUT8_BCR] = { 0x48040, 0 }, + [GCC_PCNOC_BUS_TIMEOUT9_BCR] = { 0x48048, 0 }, + [GCC_UNIPHY0_BCR] = { 0x56000, 0 }, + [GCC_UNIPHY1_BCR] = { 0x56100, 0 }, + [GCC_CMN_12GPLL_BCR] = { 0x56300, 0 }, + [GCC_QPIC_BCR] = { 0x57018, 0 }, + [GCC_MDIO_BCR] = { 0x58000, 0 }, + [GCC_WCSS_CORE_TBU_BCR] = { 0x66000, 0 }, + [GCC_WCSS_Q6_TBU_BCR] = { 0x67000, 0 }, + [GCC_USB0_TBU_BCR] = { 0x6a000, 0 }, + [GCC_PCIE0_TBU_BCR] = { 0x6b000, 0 }, + [GCC_NSS_NOC_TBU_BCR] = { 0x6e000, 0 }, + [GCC_PCIE0_BCR] = { 0x75004, 0 }, + [GCC_PCIE0_PHY_BCR] = { 0x75038, 0 }, + [GCC_PCIE0PHY_PHY_BCR] = { 0x7503c, 0 }, + [GCC_PCIE0_LINK_DOWN_BCR] = { 0x75044, 0 }, + [GCC_DCC_BCR] = { 0x77000, 0 }, + [GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR] = { 0x78000, 0 }, + [GCC_SMMU_CATS_BCR] = { 0x7c000, 0 }, + [GCC_UBI0_AXI_ARES] = { 0x68010, 0 }, + [GCC_UBI0_AHB_ARES] = { 0x68010, 1 }, + [GCC_UBI0_NC_AXI_ARES] = { 0x68010, 2 }, + [GCC_UBI0_DBG_ARES] = { 0x68010, 3 }, + [GCC_UBI0_CORE_CLAMP_ENABLE] = { 0x68010, 4 }, + [GCC_UBI0_CLKRST_CLAMP_ENABLE] = { 0x68010, 5 }, + [GCC_UBI0_UTCM_ARES] = { 0x68010, 6 }, + [GCC_UBI0_CORE_ARES] = { 0x68010, 7 }, + [GCC_NSS_CFG_ARES] = { 0x68010, 16 }, + [GCC_NSS_NOC_ARES] = { 0x68010, 18 }, + [GCC_NSS_CRYPTO_ARES] = { 0x68010, 19 }, + [GCC_NSS_CSR_ARES] = { 0x68010, 20 }, + [GCC_NSS_CE_APB_ARES] = { 0x68010, 21 }, + [GCC_NSS_CE_AXI_ARES] = { 0x68010, 22 }, + [GCC_NSSNOC_CE_APB_ARES] = { 0x68010, 23 }, + [GCC_NSSNOC_CE_AXI_ARES] = { 0x68010, 24 }, + [GCC_NSSNOC_UBI0_AHB_ARES] = { 0x68010, 25 }, + [GCC_NSSNOC_SNOC_ARES] = { 0x68010, 27 }, + [GCC_NSSNOC_CRYPTO_ARES] = { 0x68010, 28 }, + [GCC_NSSNOC_ATB_ARES] = { 0x68010, 29 }, + [GCC_NSSNOC_QOSGEN_REF_ARES] = { 0x68010, 30 }, + [GCC_NSSNOC_TIMEOUT_REF_ARES] = { 0x68010, 31 }, + [GCC_PCIE0_PIPE_ARES] = { 0x75040, 0 }, + [GCC_PCIE0_SLEEP_ARES] = { 0x75040, 1 }, + [GCC_PCIE0_CORE_STICKY_ARES] = { 0x75040, 2 }, + [GCC_PCIE0_AXI_MASTER_ARES] = { 0x75040, 3 }, + [GCC_PCIE0_AXI_SLAVE_ARES] = { 0x75040, 4 }, + [GCC_PCIE0_AHB_ARES] = { 0x75040, 5 }, + [GCC_PCIE0_AXI_MASTER_STICKY_ARES] = { 0x75040, 6 }, + [GCC_PCIE0_AXI_SLAVE_STICKY_ARES] = { 0x75040, 7 }, + [GCC_PPE_FULL_RESET] = { 0x68014, 0 }, + [GCC_UNIPHY0_SOFT_RESET] = { 0x56004, 0 }, + [GCC_UNIPHY0_XPCS_RESET] = { 0x56004, 2 }, + [GCC_UNIPHY1_SOFT_RESET] = { 0x56104, 0 }, + [GCC_UNIPHY1_XPCS_RESET] = { 0x56104, 2 }, + [GCC_EDMA_HW_RESET] = { 0x68014, 0 }, + [GCC_NSSPORT1_RESET] = { 0x68014, 0 }, + [GCC_NSSPORT2_RESET] = { 0x68014, 0 }, + [GCC_NSSPORT3_RESET] = { 0x68014, 0 }, + [GCC_NSSPORT4_RESET] = { 0x68014, 0 }, + [GCC_NSSPORT5_RESET] = { 0x68014, 0 }, + [GCC_UNIPHY0_PORT1_ARES] = { 0x56004, 0 }, + [GCC_UNIPHY0_PORT2_ARES] = { 0x56004, 0 }, + [GCC_UNIPHY0_PORT3_ARES] = { 0x56004, 0 }, + [GCC_UNIPHY0_PORT4_ARES] = { 0x56004, 0 }, + [GCC_UNIPHY0_PORT5_ARES] = { 0x56004, 0 }, + [GCC_UNIPHY0_PORT_4_5_RESET] = { 0x56004, 0 }, + [GCC_UNIPHY0_PORT_4_RESET] = { 0x56004, 0 }, + [GCC_LPASS_BCR] = {0x1F000, 0}, + [GCC_UBI32_TBU_BCR] = {0x65000, 0}, + [GCC_LPASS_TBU_BCR] = {0x6C000, 0}, + [GCC_WCSSAON_RESET] = {0x59010, 0}, + [GCC_LPASS_Q6_AXIM_ARES] = {0x1F004, 0}, + [GCC_LPASS_Q6SS_TSCTR_1TO2_ARES] = {0x1F004, 1}, + [GCC_LPASS_Q6SS_TRIG_ARES] = {0x1F004, 2}, + [GCC_LPASS_Q6_ATBM_AT_ARES] = {0x1F004, 3}, + [GCC_LPASS_Q6_PCLKDBG_ARES] = {0x1F004, 4}, + [GCC_LPASS_CORE_AXIM_ARES] = {0x1F004, 5}, + [GCC_LPASS_SNOC_CFG_ARES] = {0x1F004, 6}, + [GCC_WCSS_DBG_ARES] = {0x59008, 0}, + [GCC_WCSS_ECAHB_ARES] = {0x59008, 1}, + [GCC_WCSS_ACMT_ARES] = {0x59008, 2}, + [GCC_WCSS_DBG_BDG_ARES] = {0x59008, 3}, + [GCC_WCSS_AHB_S_ARES] = {0x59008, 4}, + [GCC_WCSS_AXI_M_ARES] = {0x59008, 5}, + [GCC_Q6SS_DBG_ARES] = {0x59110, 0}, + [GCC_Q6_AHB_S_ARES] = {0x59110, 1}, + [GCC_Q6_AHB_ARES] = {0x59110, 2}, + [GCC_Q6_AXIM2_ARES] = {0x59110, 3}, + [GCC_Q6_AXIM_ARES] = {0x59110, 4}, +}; + +static const struct of_device_id gcc_ipq6018_match_table[] = { + { .compatible = "qcom,gcc-ipq6018" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_ipq6018_match_table); + +static const struct regmap_config gcc_ipq6018_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x7fffc, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_ipq6018_desc = { + .config = &gcc_ipq6018_regmap_config, + .clks = gcc_ipq6018_clks, + .num_clks = ARRAY_SIZE(gcc_ipq6018_clks), + .resets = gcc_ipq6018_resets, + .num_resets = ARRAY_SIZE(gcc_ipq6018_resets), + .clk_hws = gcc_ipq6018_hws, + .num_clk_hws = ARRAY_SIZE(gcc_ipq6018_hws), +}; + +static int gcc_ipq6018_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &gcc_ipq6018_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* Disable SW_COLLAPSE for USB0 GDSCR */ + regmap_update_bits(regmap, 0x3e078, BIT(0), 0x0); + /* Enable SW_OVERRIDE for USB0 GDSCR */ + regmap_update_bits(regmap, 0x3e078, BIT(2), BIT(2)); + /* Disable SW_COLLAPSE for USB1 GDSCR */ + regmap_update_bits(regmap, 0x3f078, BIT(0), 0x0); + /* Enable SW_OVERRIDE for USB1 GDSCR */ + regmap_update_bits(regmap, 0x3f078, BIT(2), BIT(2)); + + /* SW Workaround for UBI Huyara PLL */ + regmap_update_bits(regmap, 0x2501c, BIT(26), BIT(26)); + + clk_alpha_pll_configure(&ubi32_pll_main, regmap, &ubi32_pll_config); + + clk_alpha_pll_configure(&nss_crypto_pll_main, regmap, + &nss_crypto_pll_config); + + return qcom_cc_really_probe(pdev, &gcc_ipq6018_desc, regmap); +} + +static struct platform_driver gcc_ipq6018_driver = { + .probe = gcc_ipq6018_probe, + .driver = { + .name = "qcom,gcc-ipq6018", + .of_match_table = gcc_ipq6018_match_table, + }, +}; + +static int __init gcc_ipq6018_init(void) +{ + return platform_driver_register(&gcc_ipq6018_driver); +} +core_initcall(gcc_ipq6018_init); + +static void __exit gcc_ipq6018_exit(void) +{ + platform_driver_unregister(&gcc_ipq6018_driver); +} +module_exit(gcc_ipq6018_exit); + +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. GCC IPQ6018 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c index d004cdaa0e39..3c3a7ff04562 100644 --- a/drivers/clk/qcom/gcc-msm8996.c +++ b/drivers/clk/qcom/gcc-msm8996.c @@ -3046,7 +3046,10 @@ static struct clk_branch gcc_usb3_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb3_clkref_clk", - .parent_names = (const char *[]){ "xo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "cxo2", + .name = "xo", + }, .num_parents = 1, .ops = &clk_branch2_ops, }, @@ -3060,7 +3063,10 @@ static struct clk_branch gcc_hdmi_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_hdmi_clkref_clk", - .parent_names = (const char *[]){ "xo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "cxo2", + .name = "xo", + }, .num_parents = 1, .ops = &clk_branch2_ops, }, @@ -3074,7 +3080,10 @@ static struct clk_branch gcc_edp_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_edp_clkref_clk", - .parent_names = (const char *[]){ "xo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "cxo2", + .name = "xo", + }, .num_parents = 1, .ops = &clk_branch2_ops, }, @@ -3088,7 +3097,10 @@ static struct clk_branch gcc_ufs_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ufs_clkref_clk", - .parent_names = (const char *[]){ "xo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "cxo2", + .name = "xo", + }, .num_parents = 1, .ops = &clk_branch2_ops, }, @@ -3102,7 +3114,10 @@ static struct clk_branch gcc_pcie_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_pcie_clkref_clk", - .parent_names = (const char *[]){ "xo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "cxo2", + .name = "xo", + }, .num_parents = 1, .ops = &clk_branch2_ops, }, @@ -3116,7 +3131,10 @@ static struct clk_branch gcc_rx2_usb2_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_rx2_usb2_clkref_clk", - .parent_names = (const char *[]){ "xo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "cxo2", + .name = "xo", + }, .num_parents = 1, .ops = &clk_branch2_ops, }, @@ -3130,7 +3148,10 @@ static struct clk_branch gcc_rx1_usb2_clkref_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_rx1_usb2_clkref_clk", - .parent_names = (const char *[]){ "xo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "cxo2", + .name = "xo", + }, .num_parents = 1, .ops = &clk_branch2_ops, }, diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c index cf31b5d03270..df1d7056436c 100644 --- a/drivers/clk/qcom/gcc-msm8998.c +++ b/drivers/clk/qcom/gcc-msm8998.c @@ -1996,6 +1996,19 @@ static struct clk_branch gcc_gp3_clk = { }, }; +static struct clk_branch gcc_bimc_gfx_clk = { + .halt_reg = 0x46040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x46040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_bimc_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_gpu_bimc_gfx_clk = { .halt_reg = 0x71010, .halt_check = BRANCH_HALT, @@ -2810,6 +2823,7 @@ static struct clk_regmap *gcc_msm8998_clocks[] = { [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_BIMC_GFX_CLK] = &gcc_bimc_gfx_clk.clkr, [GCC_GPU_BIMC_GFX_CLK] = &gcc_gpu_bimc_gfx_clk.clkr, [GCC_GPU_BIMC_GFX_SRC_CLK] = &gcc_gpu_bimc_gfx_src_clk.clkr, [GCC_GPU_CFG_AHB_CLK] = &gcc_gpu_cfg_ahb_clk.clkr, diff --git a/drivers/clk/qcom/gcc-qcs404.c b/drivers/clk/qcom/gcc-qcs404.c index 9b0c4ce2ef4e..46d314d69250 100644 --- a/drivers/clk/qcom/gcc-qcs404.c +++ b/drivers/clk/qcom/gcc-qcs404.c @@ -330,7 +330,7 @@ static struct clk_alpha_pll gpll0_ao_out_main = { .parent_names = (const char *[]){ "cxo" }, .num_parents = 1, .flags = CLK_IS_CRITICAL, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_ops, }, }, }; diff --git a/drivers/clk/qcom/gpucc-sc7180.c b/drivers/clk/qcom/gpucc-sc7180.c new file mode 100644 index 000000000000..a96c0b945de2 --- /dev/null +++ b/drivers/clk/qcom/gpucc-sc7180.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include <dt-bindings/clock/qcom,gpucc-sc7180.h> + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "common.h" +#include "gdsc.h" + +#define CX_GMU_CBCR_SLEEP_MASK 0xF +#define CX_GMU_CBCR_SLEEP_SHIFT 4 +#define CX_GMU_CBCR_WAKE_MASK 0xF +#define CX_GMU_CBCR_WAKE_SHIFT 8 +#define CLK_DIS_WAIT_SHIFT 12 +#define CLK_DIS_WAIT_MASK (0xf << CLK_DIS_WAIT_SHIFT) + +enum { + P_BI_TCXO, + P_CORE_BI_PLL_TEST_SE, + P_GPLL0_OUT_MAIN, + P_GPLL0_OUT_MAIN_DIV, + P_GPU_CC_PLL1_OUT_EVEN, + P_GPU_CC_PLL1_OUT_MAIN, + P_GPU_CC_PLL1_OUT_ODD, +}; + +static const struct pll_vco fabia_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct clk_alpha_pll gpu_cc_pll1 = { + .offset = 0x100, + .vco_table = fabia_vco, + .num_vco = ARRAY_SIZE(fabia_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gpu_cc_pll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fabia_ops, + }, + }, +}; + +static const struct parent_map gpu_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GPU_CC_PLL1_OUT_MAIN, 3 }, + { P_GPLL0_OUT_MAIN, 5 }, + { P_GPLL0_OUT_MAIN_DIV, 6 }, +}; + +static const struct clk_parent_data gpu_cc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gpu_cc_pll1.clkr.hw }, + { .fw_name = "gcc_gpu_gpll0_clk_src" }, + { .fw_name = "gcc_gpu_gpll0_div_clk_src" }, +}; + +static const struct freq_tbl ftbl_gpu_cc_gmu_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN_DIV, 1.5, 0, 0), + { } +}; + +static struct clk_rcg2 gpu_cc_gmu_clk_src = { + .cmd_rcgr = 0x1120, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpu_cc_parent_map_0, + .freq_tbl = ftbl_gpu_cc_gmu_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpu_cc_gmu_clk_src", + .parent_data = gpu_cc_parent_data_0, + .num_parents = ARRAY_SIZE(gpu_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_branch gpu_cc_crc_ahb_clk = { + .halt_reg = 0x107c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x107c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpu_cc_crc_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_cx_gmu_clk = { + .halt_reg = 0x1098, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1098, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpu_cc_cx_gmu_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gpu_cc_gmu_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_cx_snoc_dvm_clk = { + .halt_reg = 0x108c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x108c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpu_cc_cx_snoc_dvm_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_cxo_aon_clk = { + .halt_reg = 0x1004, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpu_cc_cxo_aon_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_cxo_clk = { + .halt_reg = 0x109c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x109c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpu_cc_cxo_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc cx_gdsc = { + .gdscr = 0x106c, + .gds_hw_ctrl = 0x1540, + .pd = { + .name = "cx_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc *gpu_cc_sc7180_gdscs[] = { + [CX_GDSC] = &cx_gdsc, +}; + +static struct clk_regmap *gpu_cc_sc7180_clocks[] = { + [GPU_CC_CXO_CLK] = &gpu_cc_cxo_clk.clkr, + [GPU_CC_CRC_AHB_CLK] = &gpu_cc_crc_ahb_clk.clkr, + [GPU_CC_CX_GMU_CLK] = &gpu_cc_cx_gmu_clk.clkr, + [GPU_CC_CX_SNOC_DVM_CLK] = &gpu_cc_cx_snoc_dvm_clk.clkr, + [GPU_CC_CXO_AON_CLK] = &gpu_cc_cxo_aon_clk.clkr, + [GPU_CC_GMU_CLK_SRC] = &gpu_cc_gmu_clk_src.clkr, + [GPU_CC_PLL1] = &gpu_cc_pll1.clkr, +}; + +static const struct regmap_config gpu_cc_sc7180_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x8008, + .fast_io = true, +}; + +static const struct qcom_cc_desc gpu_cc_sc7180_desc = { + .config = &gpu_cc_sc7180_regmap_config, + .clks = gpu_cc_sc7180_clocks, + .num_clks = ARRAY_SIZE(gpu_cc_sc7180_clocks), + .gdscs = gpu_cc_sc7180_gdscs, + .num_gdscs = ARRAY_SIZE(gpu_cc_sc7180_gdscs), +}; + +static const struct of_device_id gpu_cc_sc7180_match_table[] = { + { .compatible = "qcom,sc7180-gpucc" }, + { } +}; +MODULE_DEVICE_TABLE(of, gpu_cc_sc7180_match_table); + +static int gpu_cc_sc7180_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + struct alpha_pll_config gpu_cc_pll_config = {}; + unsigned int value, mask; + + regmap = qcom_cc_map(pdev, &gpu_cc_sc7180_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* 360MHz Configuration */ + gpu_cc_pll_config.l = 0x12; + gpu_cc_pll_config.alpha = 0xc000; + gpu_cc_pll_config.config_ctl_val = 0x20485699; + gpu_cc_pll_config.config_ctl_hi_val = 0x00002067; + gpu_cc_pll_config.user_ctl_val = 0x00000001; + gpu_cc_pll_config.user_ctl_hi_val = 0x00004805; + gpu_cc_pll_config.test_ctl_hi_val = 0x40000000; + + clk_fabia_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll_config); + + /* Recommended WAKEUP/SLEEP settings for the gpu_cc_cx_gmu_clk */ + mask = CX_GMU_CBCR_WAKE_MASK << CX_GMU_CBCR_WAKE_SHIFT; + mask |= CX_GMU_CBCR_SLEEP_MASK << CX_GMU_CBCR_SLEEP_SHIFT; + value = 0xF << CX_GMU_CBCR_WAKE_SHIFT | 0xF << CX_GMU_CBCR_SLEEP_SHIFT; + regmap_update_bits(regmap, 0x1098, mask, value); + + /* Configure clk_dis_wait for gpu_cx_gdsc */ + regmap_update_bits(regmap, 0x106c, CLK_DIS_WAIT_MASK, + 8 << CLK_DIS_WAIT_SHIFT); + + return qcom_cc_really_probe(pdev, &gpu_cc_sc7180_desc, regmap); +} + +static struct platform_driver gpu_cc_sc7180_driver = { + .probe = gpu_cc_sc7180_probe, + .driver = { + .name = "sc7180-gpucc", + .of_match_table = gpu_cc_sc7180_match_table, + }, +}; + +static int __init gpu_cc_sc7180_init(void) +{ + return platform_driver_register(&gpu_cc_sc7180_driver); +} +subsys_initcall(gpu_cc_sc7180_init); + +static void __exit gpu_cc_sc7180_exit(void) +{ + platform_driver_unregister(&gpu_cc_sc7180_driver); +} +module_exit(gpu_cc_sc7180_exit); + +MODULE_DESCRIPTION("QTI GPU_CC SC7180 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/hfpll.c b/drivers/clk/qcom/hfpll.c index a6de7101430c..5ff7f5a60620 100644 --- a/drivers/clk/qcom/hfpll.c +++ b/drivers/clk/qcom/hfpll.c @@ -53,10 +53,18 @@ static int qcom_hfpll_probe(struct platform_device *pdev) struct regmap *regmap; struct clk_hfpll *h; struct clk_init_data init = { - .parent_names = (const char *[]){ "xo" }, .num_parents = 1, .ops = &clk_ops_hfpll, + /* + * rather than marking the clock critical and forcing the clock + * to be always enabled, we make sure that the clock is not + * disabled: the firmware remains responsible of enabling this + * clock (for more info check the commit log) + */ + .flags = CLK_IGNORE_UNUSED, }; + int ret; + struct clk_parent_data pdata = { .index = 0 }; h = devm_kzalloc(dev, sizeof(*h), GFP_KERNEL); if (!h) @@ -75,11 +83,20 @@ static int qcom_hfpll_probe(struct platform_device *pdev) 0, &init.name)) return -ENODEV; + init.parent_data = &pdata; + h->d = &hdata; h->clkr.hw.init = &init; spin_lock_init(&h->lock); - return devm_clk_register_regmap(&pdev->dev, &h->clkr); + ret = devm_clk_register_regmap(dev, &h->clkr); + if (ret) { + dev_err(dev, "failed to register regmap clock: %d\n", ret); + return ret; + } + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &h->clkr.hw); } static struct platform_driver qcom_hfpll_driver = { diff --git a/drivers/clk/qcom/mmcc-msm8974.c b/drivers/clk/qcom/mmcc-msm8974.c index bcb0a397ef91..015426262d08 100644 --- a/drivers/clk/qcom/mmcc-msm8974.c +++ b/drivers/clk/qcom/mmcc-msm8974.c @@ -452,18 +452,6 @@ static struct clk_rcg2 mdp_clk_src = { }, }; -static struct clk_rcg2 gfx3d_clk_src = { - .cmd_rcgr = 0x4000, - .hid_width = 5, - .parent_map = mmcc_xo_mmpll0_1_2_gpll0_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gfx3d_clk_src", - .parent_names = mmcc_xo_mmpll0_1_2_gpll0, - .num_parents = 5, - .ops = &clk_rcg2_ops, - }, -}; - static struct freq_tbl ftbl_camss_jpeg_jpeg0_2_clk[] = { F(75000000, P_GPLL0, 8, 0, 0), F(133330000, P_GPLL0, 4.5, 0, 0), @@ -2411,7 +2399,6 @@ static struct clk_regmap *mmcc_msm8974_clocks[] = { [VFE0_CLK_SRC] = &vfe0_clk_src.clkr, [VFE1_CLK_SRC] = &vfe1_clk_src.clkr, [MDP_CLK_SRC] = &mdp_clk_src.clkr, - [GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr, [JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr, [JPEG1_CLK_SRC] = &jpeg1_clk_src.clkr, [JPEG2_CLK_SRC] = &jpeg2_clk_src.clkr, diff --git a/drivers/clk/qcom/mmcc-msm8998.c b/drivers/clk/qcom/mmcc-msm8998.c new file mode 100644 index 000000000000..dd68983fe22e --- /dev/null +++ b/drivers/clk/qcom/mmcc-msm8998.c @@ -0,0 +1,2913 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/err.h> +#include <linux/platform_device.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/clk-provider.h> +#include <linux/regmap.h> +#include <linux/reset-controller.h> + +#include <dt-bindings/clock/qcom,mmcc-msm8998.h> + +#include "common.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-alpha-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "reset.h" +#include "gdsc.h" + +enum { + P_XO, + P_GPLL0, + P_GPLL0_DIV, + P_MMPLL0_OUT_EVEN, + P_MMPLL1_OUT_EVEN, + P_MMPLL3_OUT_EVEN, + P_MMPLL4_OUT_EVEN, + P_MMPLL5_OUT_EVEN, + P_MMPLL6_OUT_EVEN, + P_MMPLL7_OUT_EVEN, + P_MMPLL10_OUT_EVEN, + P_DSI0PLL, + P_DSI1PLL, + P_DSI0PLL_BYTE, + P_DSI1PLL_BYTE, + P_HDMIPLL, + P_DPVCO, + P_DPLINK, + P_CORE_BI_PLL_TEST_SE, +}; + +static struct clk_fixed_factor gpll0_div = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "mmss_gpll0_div", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "gpll0", + .name = "gpll0" + }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + }, +}; + +static const struct clk_div_table post_div_table_fabia_even[] = { + { 0x0, 1 }, + { 0x1, 2 }, + { 0x3, 4 }, + { 0x7, 8 }, + { } +}; + +static struct clk_alpha_pll mmpll0 = { + .offset = 0xc000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr = { + .enable_reg = 0x1e0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll0_out_even = { + .offset = 0xc000, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll0_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll0.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static struct clk_alpha_pll mmpll1 = { + .offset = 0xc050, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr = { + .enable_reg = 0x1e0, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "mmpll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll1_out_even = { + .offset = 0xc050, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll1_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll1.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static struct clk_alpha_pll mmpll3 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll3", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll3_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll3_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll3.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static struct clk_alpha_pll mmpll4 = { + .offset = 0x50, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll4_out_even = { + .offset = 0x50, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll4_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll4.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static struct clk_alpha_pll mmpll5 = { + .offset = 0xa0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll5", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll5_out_even = { + .offset = 0xa0, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll5_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll5.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static struct clk_alpha_pll mmpll6 = { + .offset = 0xf0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll6", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll6_out_even = { + .offset = 0xf0, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll6_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll6.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static struct clk_alpha_pll mmpll7 = { + .offset = 0x140, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll7", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll7_out_even = { + .offset = 0x140, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll7_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll7.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static struct clk_alpha_pll mmpll10 = { + .offset = 0x190, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll10", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, +}; + +static struct clk_alpha_pll_postdiv mmpll10_out_even = { + .offset = 0x190, + .post_div_shift = 8, + .post_div_table = post_div_table_fabia_even, + .num_post_div = ARRAY_SIZE(post_div_table_fabia_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll10_out_even", + .parent_hws = (const struct clk_hw *[]){ &mmpll10.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_fabia_ops, + }, +}; + +static const struct parent_map mmss_xo_hdmi_map[] = { + { P_XO, 0 }, + { P_HDMIPLL, 1 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_hdmi[] = { + { .fw_name = "xo", .name = "xo" }, + { .fw_name = "hdmipll", .name = "hdmipll" }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_dsi0pll_dsi1pll_map[] = { + { P_XO, 0 }, + { P_DSI0PLL, 1 }, + { P_DSI1PLL, 2 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_dsi0pll_dsi1pll[] = { + { .fw_name = "xo", .name = "xo" }, + { .fw_name = "dsi0dsi", .name = "dsi0dsi" }, + { .fw_name = "dsi1dsi", .name = "dsi1dsi" }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_dsibyte_map[] = { + { P_XO, 0 }, + { P_DSI0PLL_BYTE, 1 }, + { P_DSI1PLL_BYTE, 2 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_dsibyte[] = { + { .fw_name = "xo", .name = "xo" }, + { .fw_name = "dsi0byte", .name = "dsi0byte" }, + { .fw_name = "dsi1byte", .name = "dsi1byte" }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_dp_map[] = { + { P_XO, 0 }, + { P_DPLINK, 1 }, + { P_DPVCO, 2 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_dp[] = { + { .fw_name = "xo", .name = "xo" }, + { .fw_name = "dplink", .name = "dplink" }, + { .fw_name = "dpvco", .name = "dpvco" }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_mmpll0_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0_OUT_EVEN, 1 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_mmpll0_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .hw = &mmpll0_out_even.clkr.hw }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0_OUT_EVEN, 1 }, + { P_MMPLL1_OUT_EVEN, 2 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .hw = &mmpll0_out_even.clkr.hw }, + { .hw = &mmpll1_out_even.clkr.hw }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0_OUT_EVEN, 1 }, + { P_MMPLL5_OUT_EVEN, 2 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .hw = &mmpll0_out_even.clkr.hw }, + { .hw = &mmpll5_out_even.clkr.hw }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0_OUT_EVEN, 1 }, + { P_MMPLL3_OUT_EVEN, 3 }, + { P_MMPLL6_OUT_EVEN, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .hw = &mmpll0_out_even.clkr.hw }, + { .hw = &mmpll3_out_even.clkr.hw }, + { .hw = &mmpll6_out_even.clkr.hw }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL4_OUT_EVEN, 1 }, + { P_MMPLL7_OUT_EVEN, 2 }, + { P_MMPLL10_OUT_EVEN, 3 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .hw = &mmpll4_out_even.clkr.hw }, + { .hw = &mmpll7_out_even.clkr.hw }, + { .hw = &mmpll10_out_even.clkr.hw }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_mmpll0_mmpll7_mmpll10_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0_OUT_EVEN, 1 }, + { P_MMPLL7_OUT_EVEN, 2 }, + { P_MMPLL10_OUT_EVEN, 3 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_mmpll0_mmpll7_mmpll10_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .hw = &mmpll0_out_even.clkr.hw }, + { .hw = &mmpll7_out_even.clkr.hw }, + { .hw = &mmpll10_out_even.clkr.hw }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0_OUT_EVEN, 1 }, + { P_MMPLL4_OUT_EVEN, 2 }, + { P_MMPLL7_OUT_EVEN, 3 }, + { P_MMPLL10_OUT_EVEN, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 } +}; + +static const struct clk_parent_data mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div[] = { + { .fw_name = "xo", .name = "xo" }, + { .hw = &mmpll0_out_even.clkr.hw }, + { .hw = &mmpll4_out_even.clkr.hw }, + { .hw = &mmpll7_out_even.clkr.hw }, + { .hw = &mmpll10_out_even.clkr.hw }, + { .fw_name = "gpll0", .name = "gpll0" }, + { .hw = &gpll0_div.hw }, + { .fw_name = "core_bi_pll_test_se", .name = "core_bi_pll_test_se" }, +}; + +static struct clk_rcg2 byte0_clk_src = { + .cmd_rcgr = 0x2120, + .hid_width = 5, + .parent_map = mmss_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte0_clk_src", + .parent_data = mmss_xo_dsibyte, + .num_parents = 4, + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 byte1_clk_src = { + .cmd_rcgr = 0x2140, + .hid_width = 5, + .parent_map = mmss_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte1_clk_src", + .parent_data = mmss_xo_dsibyte, + .num_parents = 4, + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_cci_clk_src[] = { + F(37500000, P_GPLL0, 16, 0, 0), + F(50000000, P_GPLL0, 12, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + { } +}; + +static struct clk_rcg2 cci_clk_src = { + .cmd_rcgr = 0x3300, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_cci_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cci_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_cpp_clk_src[] = { + F(100000000, P_GPLL0, 6, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(384000000, P_MMPLL4_OUT_EVEN, 2, 0, 0), + F(404000000, P_MMPLL0_OUT_EVEN, 2, 0, 0), + F(480000000, P_MMPLL7_OUT_EVEN, 2, 0, 0), + F(576000000, P_MMPLL10_OUT_EVEN, 1, 0, 0), + F(600000000, P_GPLL0, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cpp_clk_src = { + .cmd_rcgr = 0x3640, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_cpp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cpp_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi_clk_src[] = { + F(164571429, P_MMPLL10_OUT_EVEN, 3.5, 0, 0), + F(256000000, P_MMPLL4_OUT_EVEN, 3, 0, 0), + F(274290000, P_MMPLL7_OUT_EVEN, 3.5, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(384000000, P_MMPLL4_OUT_EVEN, 2, 0, 0), + F(576000000, P_MMPLL10_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 csi0_clk_src = { + .cmd_rcgr = 0x3090, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1_clk_src = { + .cmd_rcgr = 0x3100, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2_clk_src = { + .cmd_rcgr = 0x3160, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi3_clk_src = { + .cmd_rcgr = 0x31c0, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi3_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csiphy_clk_src[] = { + F(164571429, P_MMPLL10_OUT_EVEN, 3.5, 0, 0), + F(256000000, P_MMPLL4_OUT_EVEN, 3, 0, 0), + F(274290000, P_MMPLL7_OUT_EVEN, 3.5, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(384000000, P_MMPLL4_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 csiphy_clk_src = { + .cmd_rcgr = 0x3800, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csiphy_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csiphy_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csiphytimer_clk_src[] = { + F(200000000, P_GPLL0, 3, 0, 0), + F(269333333, P_MMPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi0phytimer_clk_src = { + .cmd_rcgr = 0x3000, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csiphytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0phytimer_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1phytimer_clk_src = { + .cmd_rcgr = 0x3030, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csiphytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1phytimer_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2phytimer_clk_src = { + .cmd_rcgr = 0x3060, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csiphytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2phytimer_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_aux_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 dp_aux_clk_src = { + .cmd_rcgr = 0x2260, + .hid_width = 5, + .parent_map = mmss_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_aux_clk_src", + .parent_data = mmss_xo_gpll0_gpll0_div, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_crypto_clk_src[] = { + F(101250, P_DPLINK, 1, 5, 16), + F(168750, P_DPLINK, 1, 5, 16), + F(337500, P_DPLINK, 1, 5, 16), + { } +}; + +static struct clk_rcg2 dp_crypto_clk_src = { + .cmd_rcgr = 0x2220, + .hid_width = 5, + .parent_map = mmss_xo_dp_map, + .freq_tbl = ftbl_dp_crypto_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_crypto_clk_src", + .parent_data = mmss_xo_dp, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_link_clk_src[] = { + F(162000, P_DPLINK, 2, 0, 0), + F(270000, P_DPLINK, 2, 0, 0), + F(540000, P_DPLINK, 2, 0, 0), + { } +}; + +static struct clk_rcg2 dp_link_clk_src = { + .cmd_rcgr = 0x2200, + .hid_width = 5, + .parent_map = mmss_xo_dp_map, + .freq_tbl = ftbl_dp_link_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_link_clk_src", + .parent_data = mmss_xo_dp, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_pixel_clk_src[] = { + F(154000000, P_DPVCO, 1, 0, 0), + F(337500000, P_DPVCO, 2, 0, 0), + F(675000000, P_DPVCO, 2, 0, 0), + { } +}; + +static struct clk_rcg2 dp_pixel_clk_src = { + .cmd_rcgr = 0x2240, + .hid_width = 5, + .parent_map = mmss_xo_dp_map, + .freq_tbl = ftbl_dp_pixel_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_pixel_clk_src", + .parent_data = mmss_xo_dp, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_esc_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 esc0_clk_src = { + .cmd_rcgr = 0x2160, + .hid_width = 5, + .parent_map = mmss_xo_dsibyte_map, + .freq_tbl = ftbl_esc_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc0_clk_src", + .parent_data = mmss_xo_dsibyte, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 esc1_clk_src = { + .cmd_rcgr = 0x2180, + .hid_width = 5, + .parent_map = mmss_xo_dsibyte_map, + .freq_tbl = ftbl_esc_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc1_clk_src", + .parent_data = mmss_xo_dsibyte, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_extpclk_clk_src[] = { + { .src = P_HDMIPLL }, + { } +}; + +static struct clk_rcg2 extpclk_clk_src = { + .cmd_rcgr = 0x2060, + .hid_width = 5, + .parent_map = mmss_xo_hdmi_map, + .freq_tbl = ftbl_extpclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "extpclk_clk_src", + .parent_data = mmss_xo_hdmi, + .num_parents = 3, + .ops = &clk_byte_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_fd_core_clk_src[] = { + F(100000000, P_GPLL0, 6, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(404000000, P_MMPLL0_OUT_EVEN, 2, 0, 0), + F(480000000, P_MMPLL7_OUT_EVEN, 2, 0, 0), + F(576000000, P_MMPLL10_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 fd_core_clk_src = { + .cmd_rcgr = 0x3b00, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_fd_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "fd_core_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_hdmi_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 hdmi_clk_src = { + .cmd_rcgr = 0x2100, + .hid_width = 5, + .parent_map = mmss_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_hdmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "hdmi_clk_src", + .parent_data = mmss_xo_gpll0_gpll0_div, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_jpeg0_clk_src[] = { + F(75000000, P_GPLL0, 8, 0, 0), + F(150000000, P_GPLL0, 4, 0, 0), + F(320000000, P_MMPLL7_OUT_EVEN, 3, 0, 0), + F(480000000, P_MMPLL7_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 jpeg0_clk_src = { + .cmd_rcgr = 0x3500, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_jpeg0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "jpeg0_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_maxi_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(75000000, P_GPLL0_DIV, 4, 0, 0), + F(171428571, P_GPLL0, 3.5, 0, 0), + F(323200000, P_MMPLL0_OUT_EVEN, 2.5, 0, 0), + F(406000000, P_MMPLL1_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 maxi_clk_src = { + .cmd_rcgr = 0xf020, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div_map, + .freq_tbl = ftbl_maxi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "maxi_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mclk_clk_src[] = { + F(4800000, P_XO, 4, 0, 0), + F(6000000, P_GPLL0_DIV, 10, 1, 5), + F(8000000, P_GPLL0_DIV, 1, 2, 75), + F(9600000, P_XO, 2, 0, 0), + F(16666667, P_GPLL0_DIV, 2, 1, 9), + F(19200000, P_XO, 1, 0, 0), + F(24000000, P_GPLL0_DIV, 1, 2, 25), + F(33333333, P_GPLL0_DIV, 1, 2, 9), + F(48000000, P_GPLL0, 1, 2, 25), + F(66666667, P_GPLL0, 1, 2, 9), + { } +}; + +static struct clk_rcg2 mclk0_clk_src = { + .cmd_rcgr = 0x3360, + .hid_width = 5, + .parent_map = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk0_clk_src", + .parent_data = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk1_clk_src = { + .cmd_rcgr = 0x3390, + .hid_width = 5, + .parent_map = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk1_clk_src", + .parent_data = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk2_clk_src = { + .cmd_rcgr = 0x33c0, + .hid_width = 5, + .parent_map = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk2_clk_src", + .parent_data = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk3_clk_src = { + .cmd_rcgr = 0x33f0, + .hid_width = 5, + .parent_map = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk3_clk_src", + .parent_data = mmss_xo_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mdp_clk_src[] = { + F(85714286, P_GPLL0, 7, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(150000000, P_GPLL0, 4, 0, 0), + F(171428571, P_GPLL0, 3.5, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(275000000, P_MMPLL5_OUT_EVEN, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(330000000, P_MMPLL5_OUT_EVEN, 2.5, 0, 0), + F(412500000, P_MMPLL5_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 mdp_clk_src = { + .cmd_rcgr = 0x2040, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mdp_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_vsync_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vsync_clk_src = { + .cmd_rcgr = 0x2080, + .hid_width = 5, + .parent_map = mmss_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_vsync_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vsync_clk_src", + .parent_data = mmss_xo_gpll0_gpll0_div, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_ahb_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(40000000, P_GPLL0, 15, 0, 0), + F(80800000, P_MMPLL0_OUT_EVEN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 ahb_clk_src = { + .cmd_rcgr = 0x5000, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_gpll0_gpll0_div_map, + .freq_tbl = ftbl_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ahb_clk_src", + .parent_data = mmss_xo_mmpll0_gpll0_gpll0_div, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_axi_clk_src[] = { + F(75000000, P_GPLL0, 8, 0, 0), + F(171428571, P_GPLL0, 3.5, 0, 0), + F(240000000, P_GPLL0, 2.5, 0, 0), + F(323200000, P_MMPLL0_OUT_EVEN, 2.5, 0, 0), + F(406000000, P_MMPLL0_OUT_EVEN, 2, 0, 0), + { } +}; + +/* RO to linux */ +static struct clk_rcg2 axi_clk_src = { + .cmd_rcgr = 0xd000, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div_map, + .freq_tbl = ftbl_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "axi_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll1_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 pclk0_clk_src = { + .cmd_rcgr = 0x2000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmss_xo_dsi0pll_dsi1pll_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk0_clk_src", + .parent_data = mmss_xo_dsi0pll_dsi1pll, + .num_parents = 4, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 pclk1_clk_src = { + .cmd_rcgr = 0x2020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmss_xo_dsi0pll_dsi1pll_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk1_clk_src", + .parent_data = mmss_xo_dsi0pll_dsi1pll, + .num_parents = 4, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static const struct freq_tbl ftbl_rot_clk_src[] = { + F(171428571, P_GPLL0, 3.5, 0, 0), + F(275000000, P_MMPLL5_OUT_EVEN, 3, 0, 0), + F(330000000, P_MMPLL5_OUT_EVEN, 2.5, 0, 0), + F(412500000, P_MMPLL5_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 rot_clk_src = { + .cmd_rcgr = 0x21a0, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div_map, + .freq_tbl = ftbl_rot_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rot_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll5_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_video_core_clk_src[] = { + F(200000000, P_GPLL0, 3, 0, 0), + F(269330000, P_MMPLL0_OUT_EVEN, 3, 0, 0), + F(355200000, P_MMPLL6_OUT_EVEN, 2.5, 0, 0), + F(444000000, P_MMPLL6_OUT_EVEN, 2, 0, 0), + F(533000000, P_MMPLL3_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 video_core_clk_src = { + .cmd_rcgr = 0x1000, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div_map, + .freq_tbl = ftbl_video_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_core_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 video_subcore0_clk_src = { + .cmd_rcgr = 0x1060, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div_map, + .freq_tbl = ftbl_video_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_subcore0_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 video_subcore1_clk_src = { + .cmd_rcgr = 0x1080, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div_map, + .freq_tbl = ftbl_video_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_subcore1_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll3_mmpll6_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_vfe_clk_src[] = { + F(200000000, P_GPLL0, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(320000000, P_MMPLL7_OUT_EVEN, 3, 0, 0), + F(384000000, P_MMPLL4_OUT_EVEN, 2, 0, 0), + F(404000000, P_MMPLL0_OUT_EVEN, 2, 0, 0), + F(480000000, P_MMPLL7_OUT_EVEN, 2, 0, 0), + F(576000000, P_MMPLL10_OUT_EVEN, 1, 0, 0), + F(600000000, P_GPLL0, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vfe0_clk_src = { + .cmd_rcgr = 0x3600, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_vfe_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe0_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 vfe1_clk_src = { + .cmd_rcgr = 0x3620, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_vfe_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe1_clk_src", + .parent_data = mmss_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 8, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch misc_ahb_clk = { + .halt_reg = 0x328, + .clkr = { + .enable_reg = 0x328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "misc_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch video_core_clk = { + .halt_reg = 0x1028, + .clkr = { + .enable_reg = 0x1028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_core_clk", + .parent_hws = (const struct clk_hw *[]){ &video_core_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch video_ahb_clk = { + .halt_reg = 0x1030, + .clkr = { + .enable_reg = 0x1030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch video_axi_clk = { + .halt_reg = 0x1034, + .clkr = { + .enable_reg = 0x1034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_maxi_clk = { + .halt_reg = 0x1038, + .clkr = { + .enable_reg = 0x1038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_maxi_clk", + .parent_hws = (const struct clk_hw *[]){ &maxi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch video_subcore0_clk = { + .halt_reg = 0x1048, + .clkr = { + .enable_reg = 0x1048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_subcore0_clk", + .parent_hws = (const struct clk_hw *[]){ &video_subcore0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch video_subcore1_clk = { + .halt_reg = 0x104c, + .clkr = { + .enable_reg = 0x104c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_subcore1_clk", + .parent_hws = (const struct clk_hw *[]){ &video_subcore1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_ahb_clk = { + .halt_reg = 0x2308, + .clkr = { + .enable_reg = 0x2308, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_hdmi_dp_ahb_clk = { + .halt_reg = 0x230c, + .clkr = { + .enable_reg = 0x230c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_hdmi_dp_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_axi_clk = { + .halt_reg = 0x2310, + .clkr = { + .enable_reg = 0x2310, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_pclk0_clk = { + .halt_reg = 0x2314, + .clkr = { + .enable_reg = 0x2314, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk0_clk", + .parent_hws = (const struct clk_hw *[]){ &pclk0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_pclk1_clk = { + .halt_reg = 0x2318, + .clkr = { + .enable_reg = 0x2318, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk1_clk", + .parent_hws = (const struct clk_hw *[]){ &pclk1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_mdp_clk = { + .halt_reg = 0x231c, + .clkr = { + .enable_reg = 0x231c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_mdp_clk", + .parent_hws = (const struct clk_hw *[]){ &mdp_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_mdp_lut_clk = { + .halt_reg = 0x2320, + .clkr = { + .enable_reg = 0x2320, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_mdp_lut_clk", + .parent_hws = (const struct clk_hw *[]){ &mdp_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_extpclk_clk = { + .halt_reg = 0x2324, + .clkr = { + .enable_reg = 0x2324, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_extpclk_clk", + .parent_hws = (const struct clk_hw *[]){ &extpclk_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_vsync_clk = { + .halt_reg = 0x2328, + .clkr = { + .enable_reg = 0x2328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_vsync_clk", + .parent_hws = (const struct clk_hw *[]){ &vsync_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_hdmi_clk = { + .halt_reg = 0x2338, + .clkr = { + .enable_reg = 0x2338, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_hdmi_clk", + .parent_hws = (const struct clk_hw *[]){ &hdmi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_byte0_clk = { + .halt_reg = 0x233c, + .clkr = { + .enable_reg = 0x233c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_clk", + .parent_hws = (const struct clk_hw *[]){ &byte0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_byte1_clk = { + .halt_reg = 0x2340, + .clkr = { + .enable_reg = 0x2340, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_clk", + .parent_hws = (const struct clk_hw *[]){ &byte1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_esc0_clk = { + .halt_reg = 0x2344, + .clkr = { + .enable_reg = 0x2344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc0_clk", + .parent_hws = (const struct clk_hw *[]){ &esc0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_esc1_clk = { + .halt_reg = 0x2348, + .clkr = { + .enable_reg = 0x2348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc1_clk", + .parent_hws = (const struct clk_hw *[]){ &esc1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_rot_clk = { + .halt_reg = 0x2350, + .clkr = { + .enable_reg = 0x2350, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_rot_clk", + .parent_hws = (const struct clk_hw *[]){ &rot_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_dp_link_clk = { + .halt_reg = 0x2354, + .clkr = { + .enable_reg = 0x2354, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_link_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_link_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_dp_link_intf_clk = { + .halt_reg = 0x2358, + .clkr = { + .enable_reg = 0x2358, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_link_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_link_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_dp_crypto_clk = { + .halt_reg = 0x235c, + .clkr = { + .enable_reg = 0x235c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_crypto_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_crypto_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_dp_pixel_clk = { + .halt_reg = 0x2360, + .clkr = { + .enable_reg = 0x2360, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_pixel_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_pixel_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_dp_aux_clk = { + .halt_reg = 0x2364, + .clkr = { + .enable_reg = 0x2364, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_aux_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_aux_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_byte0_intf_clk = { + .halt_reg = 0x2374, + .clkr = { + .enable_reg = 0x2374, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &byte0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mdss_byte1_intf_clk = { + .halt_reg = 0x2378, + .clkr = { + .enable_reg = 0x2378, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &byte1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi0phytimer_clk = { + .halt_reg = 0x3024, + .clkr = { + .enable_reg = 0x3024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi1phytimer_clk = { + .halt_reg = 0x3054, + .clkr = { + .enable_reg = 0x3054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi2phytimer_clk = { + .halt_reg = 0x3084, + .clkr = { + .enable_reg = 0x3084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi0_clk = { + .halt_reg = 0x30b4, + .clkr = { + .enable_reg = 0x30b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi0_ahb_clk = { + .halt_reg = 0x30bc, + .clkr = { + .enable_reg = 0x30bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi0rdi_clk = { + .halt_reg = 0x30d4, + .clkr = { + .enable_reg = 0x30d4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi0pix_clk = { + .halt_reg = 0x30e4, + .clkr = { + .enable_reg = 0x30e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi1_clk = { + .halt_reg = 0x3124, + .clkr = { + .enable_reg = 0x3124, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi1_ahb_clk = { + .halt_reg = 0x3128, + .clkr = { + .enable_reg = 0x3128, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi1rdi_clk = { + .halt_reg = 0x3144, + .clkr = { + .enable_reg = 0x3144, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi1pix_clk = { + .halt_reg = 0x3154, + .clkr = { + .enable_reg = 0x3154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi2_clk = { + .halt_reg = 0x3184, + .clkr = { + .enable_reg = 0x3184, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi2_ahb_clk = { + .halt_reg = 0x3188, + .clkr = { + .enable_reg = 0x3188, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi2rdi_clk = { + .halt_reg = 0x31a4, + .clkr = { + .enable_reg = 0x31a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi2pix_clk = { + .halt_reg = 0x31b4, + .clkr = { + .enable_reg = 0x31b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi3_clk = { + .halt_reg = 0x31e4, + .clkr = { + .enable_reg = 0x31e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi3_ahb_clk = { + .halt_reg = 0x31e8, + .clkr = { + .enable_reg = 0x31e8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi3rdi_clk = { + .halt_reg = 0x3204, + .clkr = { + .enable_reg = 0x3204, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi3pix_clk = { + .halt_reg = 0x3214, + .clkr = { + .enable_reg = 0x3214, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_ispif_ahb_clk = { + .halt_reg = 0x3224, + .clkr = { + .enable_reg = 0x3224, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ispif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cci_clk = { + .halt_reg = 0x3344, + .clkr = { + .enable_reg = 0x3344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_clk", + .parent_hws = (const struct clk_hw *[]){ &cci_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cci_ahb_clk = { + .halt_reg = 0x3348, + .clkr = { + .enable_reg = 0x3348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_mclk0_clk = { + .halt_reg = 0x3384, + .clkr = { + .enable_reg = 0x3384, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk0_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_mclk1_clk = { + .halt_reg = 0x33b4, + .clkr = { + .enable_reg = 0x33b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk1_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_mclk2_clk = { + .halt_reg = 0x33e4, + .clkr = { + .enable_reg = 0x33e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk2_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_mclk3_clk = { + .halt_reg = 0x3414, + .clkr = { + .enable_reg = 0x3414, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk3_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_top_ahb_clk = { + .halt_reg = 0x3484, + .clkr = { + .enable_reg = 0x3484, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_top_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_ahb_clk = { + .halt_reg = 0x348c, + .clkr = { + .enable_reg = 0x348c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_micro_ahb_clk = { + .halt_reg = 0x3494, + .clkr = { + .enable_reg = 0x3494, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_micro_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_jpeg0_clk = { + .halt_reg = 0x35a8, + .clkr = { + .enable_reg = 0x35a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg0_clk", + .parent_hws = (const struct clk_hw *[]){ &jpeg0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_jpeg_ahb_clk = { + .halt_reg = 0x35b4, + .clkr = { + .enable_reg = 0x35b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_jpeg_axi_clk = { + .halt_reg = 0x35b8, + .clkr = { + .enable_reg = 0x35b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe0_ahb_clk = { + .halt_reg = 0x3668, + .clkr = { + .enable_reg = 0x3668, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_vfe1_ahb_clk = { + .halt_reg = 0x3678, + .clkr = { + .enable_reg = 0x3678, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_vfe0_clk = { + .halt_reg = 0x36a8, + .clkr = { + .enable_reg = 0x36a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_vfe1_clk = { + .halt_reg = 0x36ac, + .clkr = { + .enable_reg = 0x36ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cpp_clk = { + .halt_reg = 0x36b0, + .clkr = { + .enable_reg = 0x36b0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_clk", + .parent_hws = (const struct clk_hw *[]){ &cpp_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cpp_ahb_clk = { + .halt_reg = 0x36b4, + .clkr = { + .enable_reg = 0x36b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_vfe_vbif_ahb_clk = { + .halt_reg = 0x36b8, + .clkr = { + .enable_reg = 0x36b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vbif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_vfe_vbif_axi_clk = { + .halt_reg = 0x36bc, + .clkr = { + .enable_reg = 0x36bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vbif_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_axi_clk = { + .halt_reg = 0x36c4, + .clkr = { + .enable_reg = 0x36c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_vbif_ahb_clk = { + .halt_reg = 0x36c8, + .clkr = { + .enable_reg = 0x36c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_vbif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi_vfe0_clk = { + .halt_reg = 0x3704, + .clkr = { + .enable_reg = 0x3704, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe0_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csi_vfe1_clk = { + .halt_reg = 0x3714, + .clkr = { + .enable_reg = 0x3714, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe1_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_vfe0_stream_clk = { + .halt_reg = 0x3720, + .clkr = { + .enable_reg = 0x3720, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_stream_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_vfe1_stream_clk = { + .halt_reg = 0x3724, + .clkr = { + .enable_reg = 0x3724, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_stream_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cphy_csid0_clk = { + .halt_reg = 0x3730, + .clkr = { + .enable_reg = 0x3730, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid0_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cphy_csid1_clk = { + .halt_reg = 0x3734, + .clkr = { + .enable_reg = 0x3734, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid1_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cphy_csid2_clk = { + .halt_reg = 0x3738, + .clkr = { + .enable_reg = 0x3738, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid2_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_cphy_csid3_clk = { + .halt_reg = 0x373c, + .clkr = { + .enable_reg = 0x373c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid3_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csiphy0_clk = { + .halt_reg = 0x3740, + .clkr = { + .enable_reg = 0x3740, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy0_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csiphy1_clk = { + .halt_reg = 0x3744, + .clkr = { + .enable_reg = 0x3744, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy1_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch camss_csiphy2_clk = { + .halt_reg = 0x3748, + .clkr = { + .enable_reg = 0x3748, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy2_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch fd_core_clk = { + .halt_reg = 0x3b68, + .clkr = { + .enable_reg = 0x3b68, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "fd_core_clk", + .parent_hws = (const struct clk_hw *[]){ &fd_core_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch fd_core_uar_clk = { + .halt_reg = 0x3b6c, + .clkr = { + .enable_reg = 0x3b6c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "fd_core_uar_clk", + .parent_hws = (const struct clk_hw *[]){ &fd_core_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch fd_ahb_clk = { + .halt_reg = 0x3b74, + .clkr = { + .enable_reg = 0x3b74, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "fd_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch mnoc_ahb_clk = { + .halt_reg = 0x5024, + .clkr = { + .enable_reg = 0x5024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mnoc_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch bimc_smmu_ahb_clk = { + .halt_reg = 0xe004, + .clkr = { + .enable_reg = 0xe004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "bimc_smmu_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch bimc_smmu_axi_clk = { + .halt_reg = 0xe008, + .clkr = { + .enable_reg = 0xe008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "bimc_smmu_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mnoc_maxi_clk = { + .halt_reg = 0xf004, + .clkr = { + .enable_reg = 0xf004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mnoc_maxi_clk", + .parent_hws = (const struct clk_hw *[]){ &maxi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch vmem_maxi_clk = { + .halt_reg = 0xf064, + .clkr = { + .enable_reg = 0xf064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vmem_maxi_clk", + .parent_hws = (const struct clk_hw *[]){ &maxi_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch vmem_ahb_clk = { + .halt_reg = 0xf068, + .clkr = { + .enable_reg = 0xf068, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vmem_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_hw *mmcc_msm8998_hws[] = { + &gpll0_div.hw, +}; + +static struct gdsc video_top_gdsc = { + .gdscr = 0x1024, + .pd = { + .name = "video_top", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc video_subcore0_gdsc = { + .gdscr = 0x1040, + .pd = { + .name = "video_subcore0", + }, + .parent = &video_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc video_subcore1_gdsc = { + .gdscr = 0x1044, + .pd = { + .name = "video_subcore1", + }, + .parent = &video_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x2304, + .cxcs = (unsigned int []){ 0x2310, 0x2350, 0x231c, 0x2320 }, + .cxc_count = 4, + .pd = { + .name = "mdss", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_top_gdsc = { + .gdscr = 0x34a0, + .cxcs = (unsigned int []){ 0x35b8, 0x36c4, 0x3704, 0x3714, 0x3494, + 0x35a8, 0x3868 }, + .cxc_count = 7, + .pd = { + .name = "camss_top", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_vfe0_gdsc = { + .gdscr = 0x3664, + .pd = { + .name = "camss_vfe0", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_vfe1_gdsc = { + .gdscr = 0x3674, + .pd = { + .name = "camss_vfe1_gdsc", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_cpp_gdsc = { + .gdscr = 0x36d4, + .pd = { + .name = "camss_cpp", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc bimc_smmu_gdsc = { + .gdscr = 0xe020, + .gds_hw_ctrl = 0xe024, + .pd = { + .name = "bimc_smmu", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, +}; + +static struct clk_regmap *mmcc_msm8998_clocks[] = { + [MMPLL0] = &mmpll0.clkr, + [MMPLL0_OUT_EVEN] = &mmpll0_out_even.clkr, + [MMPLL1] = &mmpll1.clkr, + [MMPLL1_OUT_EVEN] = &mmpll1_out_even.clkr, + [MMPLL3] = &mmpll3.clkr, + [MMPLL3_OUT_EVEN] = &mmpll3_out_even.clkr, + [MMPLL4] = &mmpll4.clkr, + [MMPLL4_OUT_EVEN] = &mmpll4_out_even.clkr, + [MMPLL5] = &mmpll5.clkr, + [MMPLL5_OUT_EVEN] = &mmpll5_out_even.clkr, + [MMPLL6] = &mmpll6.clkr, + [MMPLL6_OUT_EVEN] = &mmpll6_out_even.clkr, + [MMPLL7] = &mmpll7.clkr, + [MMPLL7_OUT_EVEN] = &mmpll7_out_even.clkr, + [MMPLL10] = &mmpll10.clkr, + [MMPLL10_OUT_EVEN] = &mmpll10_out_even.clkr, + [BYTE0_CLK_SRC] = &byte0_clk_src.clkr, + [BYTE1_CLK_SRC] = &byte1_clk_src.clkr, + [CCI_CLK_SRC] = &cci_clk_src.clkr, + [CPP_CLK_SRC] = &cpp_clk_src.clkr, + [CSI0_CLK_SRC] = &csi0_clk_src.clkr, + [CSI1_CLK_SRC] = &csi1_clk_src.clkr, + [CSI2_CLK_SRC] = &csi2_clk_src.clkr, + [CSI3_CLK_SRC] = &csi3_clk_src.clkr, + [CSIPHY_CLK_SRC] = &csiphy_clk_src.clkr, + [CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr, + [CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr, + [CSI2PHYTIMER_CLK_SRC] = &csi2phytimer_clk_src.clkr, + [DP_AUX_CLK_SRC] = &dp_aux_clk_src.clkr, + [DP_CRYPTO_CLK_SRC] = &dp_crypto_clk_src.clkr, + [DP_LINK_CLK_SRC] = &dp_link_clk_src.clkr, + [DP_PIXEL_CLK_SRC] = &dp_pixel_clk_src.clkr, + [ESC0_CLK_SRC] = &esc0_clk_src.clkr, + [ESC1_CLK_SRC] = &esc1_clk_src.clkr, + [EXTPCLK_CLK_SRC] = &extpclk_clk_src.clkr, + [FD_CORE_CLK_SRC] = &fd_core_clk_src.clkr, + [HDMI_CLK_SRC] = &hdmi_clk_src.clkr, + [JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr, + [MAXI_CLK_SRC] = &maxi_clk_src.clkr, + [MCLK0_CLK_SRC] = &mclk0_clk_src.clkr, + [MCLK1_CLK_SRC] = &mclk1_clk_src.clkr, + [MCLK2_CLK_SRC] = &mclk2_clk_src.clkr, + [MCLK3_CLK_SRC] = &mclk3_clk_src.clkr, + [MDP_CLK_SRC] = &mdp_clk_src.clkr, + [VSYNC_CLK_SRC] = &vsync_clk_src.clkr, + [AHB_CLK_SRC] = &ahb_clk_src.clkr, + [AXI_CLK_SRC] = &axi_clk_src.clkr, + [PCLK0_CLK_SRC] = &pclk0_clk_src.clkr, + [PCLK1_CLK_SRC] = &pclk1_clk_src.clkr, + [ROT_CLK_SRC] = &rot_clk_src.clkr, + [VIDEO_CORE_CLK_SRC] = &video_core_clk_src.clkr, + [VIDEO_SUBCORE0_CLK_SRC] = &video_subcore0_clk_src.clkr, + [VIDEO_SUBCORE1_CLK_SRC] = &video_subcore1_clk_src.clkr, + [VFE0_CLK_SRC] = &vfe0_clk_src.clkr, + [VFE1_CLK_SRC] = &vfe1_clk_src.clkr, + [MISC_AHB_CLK] = &misc_ahb_clk.clkr, + [VIDEO_CORE_CLK] = &video_core_clk.clkr, + [VIDEO_AHB_CLK] = &video_ahb_clk.clkr, + [VIDEO_AXI_CLK] = &video_axi_clk.clkr, + [VIDEO_MAXI_CLK] = &video_maxi_clk.clkr, + [VIDEO_SUBCORE0_CLK] = &video_subcore0_clk.clkr, + [VIDEO_SUBCORE1_CLK] = &video_subcore1_clk.clkr, + [MDSS_AHB_CLK] = &mdss_ahb_clk.clkr, + [MDSS_HDMI_DP_AHB_CLK] = &mdss_hdmi_dp_ahb_clk.clkr, + [MDSS_AXI_CLK] = &mdss_axi_clk.clkr, + [MDSS_PCLK0_CLK] = &mdss_pclk0_clk.clkr, + [MDSS_PCLK1_CLK] = &mdss_pclk1_clk.clkr, + [MDSS_MDP_CLK] = &mdss_mdp_clk.clkr, + [MDSS_MDP_LUT_CLK] = &mdss_mdp_lut_clk.clkr, + [MDSS_EXTPCLK_CLK] = &mdss_extpclk_clk.clkr, + [MDSS_VSYNC_CLK] = &mdss_vsync_clk.clkr, + [MDSS_HDMI_CLK] = &mdss_hdmi_clk.clkr, + [MDSS_BYTE0_CLK] = &mdss_byte0_clk.clkr, + [MDSS_BYTE1_CLK] = &mdss_byte1_clk.clkr, + [MDSS_ESC0_CLK] = &mdss_esc0_clk.clkr, + [MDSS_ESC1_CLK] = &mdss_esc1_clk.clkr, + [MDSS_ROT_CLK] = &mdss_rot_clk.clkr, + [MDSS_DP_LINK_CLK] = &mdss_dp_link_clk.clkr, + [MDSS_DP_LINK_INTF_CLK] = &mdss_dp_link_intf_clk.clkr, + [MDSS_DP_CRYPTO_CLK] = &mdss_dp_crypto_clk.clkr, + [MDSS_DP_PIXEL_CLK] = &mdss_dp_pixel_clk.clkr, + [MDSS_DP_AUX_CLK] = &mdss_dp_aux_clk.clkr, + [MDSS_BYTE0_INTF_CLK] = &mdss_byte0_intf_clk.clkr, + [MDSS_BYTE1_INTF_CLK] = &mdss_byte1_intf_clk.clkr, + [CAMSS_CSI0PHYTIMER_CLK] = &camss_csi0phytimer_clk.clkr, + [CAMSS_CSI1PHYTIMER_CLK] = &camss_csi1phytimer_clk.clkr, + [CAMSS_CSI2PHYTIMER_CLK] = &camss_csi2phytimer_clk.clkr, + [CAMSS_CSI0_CLK] = &camss_csi0_clk.clkr, + [CAMSS_CSI0_AHB_CLK] = &camss_csi0_ahb_clk.clkr, + [CAMSS_CSI0RDI_CLK] = &camss_csi0rdi_clk.clkr, + [CAMSS_CSI0PIX_CLK] = &camss_csi0pix_clk.clkr, + [CAMSS_CSI1_CLK] = &camss_csi1_clk.clkr, + [CAMSS_CSI1_AHB_CLK] = &camss_csi1_ahb_clk.clkr, + [CAMSS_CSI1RDI_CLK] = &camss_csi1rdi_clk.clkr, + [CAMSS_CSI1PIX_CLK] = &camss_csi1pix_clk.clkr, + [CAMSS_CSI2_CLK] = &camss_csi2_clk.clkr, + [CAMSS_CSI2_AHB_CLK] = &camss_csi2_ahb_clk.clkr, + [CAMSS_CSI2RDI_CLK] = &camss_csi2rdi_clk.clkr, + [CAMSS_CSI2PIX_CLK] = &camss_csi2pix_clk.clkr, + [CAMSS_CSI3_CLK] = &camss_csi3_clk.clkr, + [CAMSS_CSI3_AHB_CLK] = &camss_csi3_ahb_clk.clkr, + [CAMSS_CSI3RDI_CLK] = &camss_csi3rdi_clk.clkr, + [CAMSS_CSI3PIX_CLK] = &camss_csi3pix_clk.clkr, + [CAMSS_ISPIF_AHB_CLK] = &camss_ispif_ahb_clk.clkr, + [CAMSS_CCI_CLK] = &camss_cci_clk.clkr, + [CAMSS_CCI_AHB_CLK] = &camss_cci_ahb_clk.clkr, + [CAMSS_MCLK0_CLK] = &camss_mclk0_clk.clkr, + [CAMSS_MCLK1_CLK] = &camss_mclk1_clk.clkr, + [CAMSS_MCLK2_CLK] = &camss_mclk2_clk.clkr, + [CAMSS_MCLK3_CLK] = &camss_mclk3_clk.clkr, + [CAMSS_TOP_AHB_CLK] = &camss_top_ahb_clk.clkr, + [CAMSS_AHB_CLK] = &camss_ahb_clk.clkr, + [CAMSS_MICRO_AHB_CLK] = &camss_micro_ahb_clk.clkr, + [CAMSS_JPEG0_CLK] = &camss_jpeg0_clk.clkr, + [CAMSS_JPEG_AHB_CLK] = &camss_jpeg_ahb_clk.clkr, + [CAMSS_JPEG_AXI_CLK] = &camss_jpeg_axi_clk.clkr, + [CAMSS_VFE0_AHB_CLK] = &camss_vfe0_ahb_clk.clkr, + [CAMSS_VFE1_AHB_CLK] = &camss_vfe1_ahb_clk.clkr, + [CAMSS_VFE0_CLK] = &camss_vfe0_clk.clkr, + [CAMSS_VFE1_CLK] = &camss_vfe1_clk.clkr, + [CAMSS_CPP_CLK] = &camss_cpp_clk.clkr, + [CAMSS_CPP_AHB_CLK] = &camss_cpp_ahb_clk.clkr, + [CAMSS_VFE_VBIF_AHB_CLK] = &camss_vfe_vbif_ahb_clk.clkr, + [CAMSS_VFE_VBIF_AXI_CLK] = &camss_vfe_vbif_axi_clk.clkr, + [CAMSS_CPP_AXI_CLK] = &camss_cpp_axi_clk.clkr, + [CAMSS_CPP_VBIF_AHB_CLK] = &camss_cpp_vbif_ahb_clk.clkr, + [CAMSS_CSI_VFE0_CLK] = &camss_csi_vfe0_clk.clkr, + [CAMSS_CSI_VFE1_CLK] = &camss_csi_vfe1_clk.clkr, + [CAMSS_VFE0_STREAM_CLK] = &camss_vfe0_stream_clk.clkr, + [CAMSS_VFE1_STREAM_CLK] = &camss_vfe1_stream_clk.clkr, + [CAMSS_CPHY_CSID0_CLK] = &camss_cphy_csid0_clk.clkr, + [CAMSS_CPHY_CSID1_CLK] = &camss_cphy_csid1_clk.clkr, + [CAMSS_CPHY_CSID2_CLK] = &camss_cphy_csid2_clk.clkr, + [CAMSS_CPHY_CSID3_CLK] = &camss_cphy_csid3_clk.clkr, + [CAMSS_CSIPHY0_CLK] = &camss_csiphy0_clk.clkr, + [CAMSS_CSIPHY1_CLK] = &camss_csiphy1_clk.clkr, + [CAMSS_CSIPHY2_CLK] = &camss_csiphy2_clk.clkr, + [FD_CORE_CLK] = &fd_core_clk.clkr, + [FD_CORE_UAR_CLK] = &fd_core_uar_clk.clkr, + [FD_AHB_CLK] = &fd_ahb_clk.clkr, + [MNOC_AHB_CLK] = &mnoc_ahb_clk.clkr, + [BIMC_SMMU_AHB_CLK] = &bimc_smmu_ahb_clk.clkr, + [BIMC_SMMU_AXI_CLK] = &bimc_smmu_axi_clk.clkr, + [MNOC_MAXI_CLK] = &mnoc_maxi_clk.clkr, + [VMEM_MAXI_CLK] = &vmem_maxi_clk.clkr, + [VMEM_AHB_CLK] = &vmem_ahb_clk.clkr, +}; + +static struct gdsc *mmcc_msm8998_gdscs[] = { + [VIDEO_TOP_GDSC] = &video_top_gdsc, + [VIDEO_SUBCORE0_GDSC] = &video_subcore0_gdsc, + [VIDEO_SUBCORE1_GDSC] = &video_subcore1_gdsc, + [MDSS_GDSC] = &mdss_gdsc, + [CAMSS_TOP_GDSC] = &camss_top_gdsc, + [CAMSS_VFE0_GDSC] = &camss_vfe0_gdsc, + [CAMSS_VFE1_GDSC] = &camss_vfe1_gdsc, + [CAMSS_CPP_GDSC] = &camss_cpp_gdsc, + [BIMC_SMMU_GDSC] = &bimc_smmu_gdsc, +}; + +static const struct qcom_reset_map mmcc_msm8998_resets[] = { + [SPDM_BCR] = { 0x200 }, + [SPDM_RM_BCR] = { 0x300 }, + [MISC_BCR] = { 0x320 }, + [VIDEO_TOP_BCR] = { 0x1020 }, + [THROTTLE_VIDEO_BCR] = { 0x1180 }, + [MDSS_BCR] = { 0x2300 }, + [THROTTLE_MDSS_BCR] = { 0x2460 }, + [CAMSS_PHY0_BCR] = { 0x3020 }, + [CAMSS_PHY1_BCR] = { 0x3050 }, + [CAMSS_PHY2_BCR] = { 0x3080 }, + [CAMSS_CSI0_BCR] = { 0x30b0 }, + [CAMSS_CSI0RDI_BCR] = { 0x30d0 }, + [CAMSS_CSI0PIX_BCR] = { 0x30e0 }, + [CAMSS_CSI1_BCR] = { 0x3120 }, + [CAMSS_CSI1RDI_BCR] = { 0x3140 }, + [CAMSS_CSI1PIX_BCR] = { 0x3150 }, + [CAMSS_CSI2_BCR] = { 0x3180 }, + [CAMSS_CSI2RDI_BCR] = { 0x31a0 }, + [CAMSS_CSI2PIX_BCR] = { 0x31b0 }, + [CAMSS_CSI3_BCR] = { 0x31e0 }, + [CAMSS_CSI3RDI_BCR] = { 0x3200 }, + [CAMSS_CSI3PIX_BCR] = { 0x3210 }, + [CAMSS_ISPIF_BCR] = { 0x3220 }, + [CAMSS_CCI_BCR] = { 0x3340 }, + [CAMSS_TOP_BCR] = { 0x3480 }, + [CAMSS_AHB_BCR] = { 0x3488 }, + [CAMSS_MICRO_BCR] = { 0x3490 }, + [CAMSS_JPEG_BCR] = { 0x35a0 }, + [CAMSS_VFE0_BCR] = { 0x3660 }, + [CAMSS_VFE1_BCR] = { 0x3670 }, + [CAMSS_VFE_VBIF_BCR] = { 0x36a0 }, + [CAMSS_CPP_TOP_BCR] = { 0x36c0 }, + [CAMSS_CPP_BCR] = { 0x36d0 }, + [CAMSS_CSI_VFE0_BCR] = { 0x3700 }, + [CAMSS_CSI_VFE1_BCR] = { 0x3710 }, + [CAMSS_FD_BCR] = { 0x3b60 }, + [THROTTLE_CAMSS_BCR] = { 0x3c30 }, + [MNOCAHB_BCR] = { 0x5020 }, + [MNOCAXI_BCR] = { 0xd020 }, + [BMIC_SMMU_BCR] = { 0xe000 }, + [MNOC_MAXI_BCR] = { 0xf000 }, + [VMEM_BCR] = { 0xf060 }, + [BTO_BCR] = { 0x10004 }, +}; + +static const struct regmap_config mmcc_msm8998_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x10004, + .fast_io = true, +}; + +static const struct qcom_cc_desc mmcc_msm8998_desc = { + .config = &mmcc_msm8998_regmap_config, + .clks = mmcc_msm8998_clocks, + .num_clks = ARRAY_SIZE(mmcc_msm8998_clocks), + .resets = mmcc_msm8998_resets, + .num_resets = ARRAY_SIZE(mmcc_msm8998_resets), + .gdscs = mmcc_msm8998_gdscs, + .num_gdscs = ARRAY_SIZE(mmcc_msm8998_gdscs), + .clk_hws = mmcc_msm8998_hws, + .num_clk_hws = ARRAY_SIZE(mmcc_msm8998_hws), +}; + +static const struct of_device_id mmcc_msm8998_match_table[] = { + { .compatible = "qcom,mmcc-msm8998" }, + { } +}; +MODULE_DEVICE_TABLE(of, mmcc_msm8998_match_table); + +static int mmcc_msm8998_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &mmcc_msm8998_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + return qcom_cc_really_probe(pdev, &mmcc_msm8998_desc, regmap); +} + +static struct platform_driver mmcc_msm8998_driver = { + .probe = mmcc_msm8998_probe, + .driver = { + .name = "mmcc-msm8998", + .of_match_table = mmcc_msm8998_match_table, + }, +}; +module_platform_driver(mmcc_msm8998_driver); + +MODULE_DESCRIPTION("QCOM MMCC MSM8998 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/videocc-sc7180.c b/drivers/clk/qcom/videocc-sc7180.c new file mode 100644 index 000000000000..c363c3cc544e --- /dev/null +++ b/drivers/clk/qcom/videocc-sc7180.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include <dt-bindings/clock/qcom,videocc-sc7180.h> + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "common.h" +#include "gdsc.h" + +enum { + P_BI_TCXO, + P_CHIP_SLEEP_CLK, + P_CORE_BI_PLL_TEST_SE, + P_VIDEO_PLL0_OUT_EVEN, + P_VIDEO_PLL0_OUT_MAIN, + P_VIDEO_PLL0_OUT_ODD, +}; + +static const struct pll_vco fabia_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct clk_alpha_pll video_pll0 = { + .offset = 0x42c, + .vco_table = fabia_vco, + .num_vco = ARRAY_SIZE(fabia_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "video_pll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fabia_ops, + }, + }, +}; + +static const struct parent_map video_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_VIDEO_PLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data video_cc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &video_pll0.clkr.hw }, +}; + +static const struct freq_tbl ftbl_video_cc_venus_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(150000000, P_VIDEO_PLL0_OUT_MAIN, 4, 0, 0), + F(270000000, P_VIDEO_PLL0_OUT_MAIN, 2.5, 0, 0), + F(340000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + F(434000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + F(500000000, P_VIDEO_PLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 video_cc_venus_clk_src = { + .cmd_rcgr = 0x7f0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = video_cc_parent_map_1, + .freq_tbl = ftbl_video_cc_venus_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_cc_venus_clk_src", + .parent_data = video_cc_parent_data_1, + .num_parents = ARRAY_SIZE(video_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_branch video_cc_vcodec0_axi_clk = { + .halt_reg = 0x9ec, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9ec, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_vcodec0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_vcodec0_core_clk = { + .halt_reg = 0x890, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x890, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_vcodec0_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_venus_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_venus_ahb_clk = { + .halt_reg = 0xa4c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa4c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_venus_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_venus_ctl_axi_clk = { + .halt_reg = 0x9cc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_venus_ctl_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_cc_venus_ctl_core_clk = { + .halt_reg = 0x850, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x850, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_venus_ctl_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_venus_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc venus_gdsc = { + .gdscr = 0x814, + .pd = { + .name = "venus_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc vcodec0_gdsc = { + .gdscr = 0x874, + .pd = { + .name = "vcodec0_gdsc", + }, + .flags = HW_CTRL, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct clk_regmap *video_cc_sc7180_clocks[] = { + [VIDEO_CC_VCODEC0_AXI_CLK] = &video_cc_vcodec0_axi_clk.clkr, + [VIDEO_CC_VCODEC0_CORE_CLK] = &video_cc_vcodec0_core_clk.clkr, + [VIDEO_CC_VENUS_AHB_CLK] = &video_cc_venus_ahb_clk.clkr, + [VIDEO_CC_VENUS_CLK_SRC] = &video_cc_venus_clk_src.clkr, + [VIDEO_CC_VENUS_CTL_AXI_CLK] = &video_cc_venus_ctl_axi_clk.clkr, + [VIDEO_CC_VENUS_CTL_CORE_CLK] = &video_cc_venus_ctl_core_clk.clkr, + [VIDEO_PLL0] = &video_pll0.clkr, +}; + +static struct gdsc *video_cc_sc7180_gdscs[] = { + [VENUS_GDSC] = &venus_gdsc, + [VCODEC0_GDSC] = &vcodec0_gdsc, +}; + +static const struct regmap_config video_cc_sc7180_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0xb94, + .fast_io = true, +}; + +static const struct qcom_cc_desc video_cc_sc7180_desc = { + .config = &video_cc_sc7180_regmap_config, + .clks = video_cc_sc7180_clocks, + .num_clks = ARRAY_SIZE(video_cc_sc7180_clocks), + .gdscs = video_cc_sc7180_gdscs, + .num_gdscs = ARRAY_SIZE(video_cc_sc7180_gdscs), +}; + +static const struct of_device_id video_cc_sc7180_match_table[] = { + { .compatible = "qcom,sc7180-videocc" }, + { } +}; +MODULE_DEVICE_TABLE(of, video_cc_sc7180_match_table); + +static int video_cc_sc7180_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + struct alpha_pll_config video_pll0_config = {}; + + regmap = qcom_cc_map(pdev, &video_cc_sc7180_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + video_pll0_config.l = 0x1f; + video_pll0_config.alpha = 0x4000; + video_pll0_config.user_ctl_val = 0x00000001; + video_pll0_config.user_ctl_hi_val = 0x00004805; + + clk_fabia_pll_configure(&video_pll0, regmap, &video_pll0_config); + + /* Keep VIDEO_CC_XO_CLK ALWAYS-ON */ + regmap_update_bits(regmap, 0x984, 0x1, 0x1); + + return qcom_cc_really_probe(pdev, &video_cc_sc7180_desc, regmap); +} + +static struct platform_driver video_cc_sc7180_driver = { + .probe = video_cc_sc7180_probe, + .driver = { + .name = "sc7180-videocc", + .of_match_table = video_cc_sc7180_match_table, + }, +}; + +static int __init video_cc_sc7180_init(void) +{ + return platform_driver_register(&video_cc_sc7180_driver); +} +subsys_initcall(video_cc_sc7180_init); + +static void __exit video_cc_sc7180_exit(void) +{ + platform_driver_unregister(&video_cc_sc7180_driver); +} +module_exit(video_cc_sc7180_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("QTI VIDEOCC SC7180 Driver"); diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig index 4cd846bc98cc..250d8165167a 100644 --- a/drivers/clk/renesas/Kconfig +++ b/drivers/clk/renesas/Kconfig @@ -20,8 +20,8 @@ config CLK_RENESAS select CLK_R8A7791 if ARCH_R8A7791 || ARCH_R8A7793 select CLK_R8A7792 if ARCH_R8A7792 select CLK_R8A7794 if ARCH_R8A7794 - select CLK_R8A7795 if ARCH_R8A7795 - select CLK_R8A77960 if ARCH_R8A77960 || ARCH_R8A7796 + select CLK_R8A7795 if ARCH_R8A77950 || ARCH_R8A77951 || ARCH_R8A7795 + select CLK_R8A77960 if ARCH_R8A77960 select CLK_R8A77961 if ARCH_R8A77961 select CLK_R8A77965 if ARCH_R8A77965 select CLK_R8A77970 if ARCH_R8A77970 diff --git a/drivers/clk/renesas/r7s9210-cpg-mssr.c b/drivers/clk/renesas/r7s9210-cpg-mssr.c index cf65d4e0e116..443bff08df4c 100644 --- a/drivers/clk/renesas/r7s9210-cpg-mssr.c +++ b/drivers/clk/renesas/r7s9210-cpg-mssr.c @@ -93,6 +93,7 @@ static const struct mssr_mod_clk r7s9210_mod_clks[] __initconst = { DEF_MOD_STB("ether1", 64, R7S9210_CLK_B), DEF_MOD_STB("ether0", 65, R7S9210_CLK_B), + DEF_MOD_STB("spibsc", 83, R7S9210_CLK_P1), DEF_MOD_STB("i2c3", 84, R7S9210_CLK_P1), DEF_MOD_STB("i2c2", 85, R7S9210_CLK_P1), DEF_MOD_STB("i2c1", 86, R7S9210_CLK_P1), diff --git a/drivers/clk/renesas/rcar-gen2-cpg.h b/drivers/clk/renesas/rcar-gen2-cpg.h index db2f57ef2f99..bdcd4a38d48d 100644 --- a/drivers/clk/renesas/rcar-gen2-cpg.h +++ b/drivers/clk/renesas/rcar-gen2-cpg.h @@ -24,10 +24,10 @@ enum rcar_gen2_clk_types { }; struct rcar_gen2_cpg_pll_config { - unsigned int extal_div; - unsigned int pll1_mult; - unsigned int pll3_mult; - unsigned int pll0_mult; /* leave as zero if PLL0CR exists */ + u8 extal_div; + u8 pll1_mult; + u8 pll3_mult; + u8 pll0_mult; /* leave as zero if PLL0CR exists */ }; struct clk *rcar_gen2_cpg_clk_register(struct device *dev, diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c index c97b647db9b6..488f8b3980c5 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.c +++ b/drivers/clk/renesas/rcar-gen3-cpg.c @@ -470,7 +470,8 @@ static struct clk * __init cpg_rpc_clk_register(const char *name, clk = clk_register_composite(NULL, name, &parent_name, 1, NULL, NULL, &rpc->div.hw, &clk_divider_ops, - &rpc->gate.hw, &clk_gate_ops, 0); + &rpc->gate.hw, &clk_gate_ops, + CLK_SET_RATE_PARENT); if (IS_ERR(clk)) { kfree(rpc); return clk; @@ -506,7 +507,8 @@ static struct clk * __init cpg_rpcd2_clk_register(const char *name, clk = clk_register_composite(NULL, name, &parent_name, 1, NULL, NULL, &rpcd2->fixed.hw, &clk_fixed_factor_ops, - &rpcd2->gate.hw, &clk_gate_ops, 0); + &rpcd2->gate.hw, &clk_gate_ops, + CLK_SET_RATE_PARENT); if (IS_ERR(clk)) kfree(rpcd2); diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c index 198417d56300..10560d963baf 100644 --- a/drivers/clk/rockchip/clk-pll.c +++ b/drivers/clk/rockchip/clk-pll.c @@ -282,7 +282,7 @@ static int rockchip_rk3036_pll_is_enabled(struct clk_hw *hw) return !(pllcon & RK3036_PLLCON1_PWRDOWN); } -static void rockchip_rk3036_pll_init(struct clk_hw *hw) +static int rockchip_rk3036_pll_init(struct clk_hw *hw) { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); const struct rockchip_pll_rate_table *rate; @@ -290,14 +290,14 @@ static void rockchip_rk3036_pll_init(struct clk_hw *hw) unsigned long drate; if (!(pll->flags & ROCKCHIP_PLL_SYNC_RATE)) - return; + return 0; drate = clk_hw_get_rate(hw); rate = rockchip_get_pll_settings(pll, drate); /* when no rate setting for the current rate, rely on clk_set_rate */ if (!rate) - return; + return 0; rockchip_rk3036_pll_get_params(pll, &cur); @@ -319,13 +319,15 @@ static void rockchip_rk3036_pll_init(struct clk_hw *hw) if (!parent) { pr_warn("%s: parent of %s not available\n", __func__, __clk_get_name(hw->clk)); - return; + return 0; } pr_debug("%s: pll %s: rate params do not match rate table, adjusting\n", __func__, __clk_get_name(hw->clk)); rockchip_rk3036_pll_set_params(pll, rate); } + + return 0; } static const struct clk_ops rockchip_rk3036_pll_clk_norate_ops = { @@ -515,7 +517,7 @@ static int rockchip_rk3066_pll_is_enabled(struct clk_hw *hw) return !(pllcon & RK3066_PLLCON3_PWRDOWN); } -static void rockchip_rk3066_pll_init(struct clk_hw *hw) +static int rockchip_rk3066_pll_init(struct clk_hw *hw) { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); const struct rockchip_pll_rate_table *rate; @@ -523,14 +525,14 @@ static void rockchip_rk3066_pll_init(struct clk_hw *hw) unsigned long drate; if (!(pll->flags & ROCKCHIP_PLL_SYNC_RATE)) - return; + return 0; drate = clk_hw_get_rate(hw); rate = rockchip_get_pll_settings(pll, drate); /* when no rate setting for the current rate, rely on clk_set_rate */ if (!rate) - return; + return 0; rockchip_rk3066_pll_get_params(pll, &cur); @@ -543,6 +545,8 @@ static void rockchip_rk3066_pll_init(struct clk_hw *hw) __func__, clk_hw_get_name(hw)); rockchip_rk3066_pll_set_params(pll, rate); } + + return 0; } static const struct clk_ops rockchip_rk3066_pll_clk_norate_ops = { @@ -761,7 +765,7 @@ static int rockchip_rk3399_pll_is_enabled(struct clk_hw *hw) return !(pllcon & RK3399_PLLCON3_PWRDOWN); } -static void rockchip_rk3399_pll_init(struct clk_hw *hw) +static int rockchip_rk3399_pll_init(struct clk_hw *hw) { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); const struct rockchip_pll_rate_table *rate; @@ -769,14 +773,14 @@ static void rockchip_rk3399_pll_init(struct clk_hw *hw) unsigned long drate; if (!(pll->flags & ROCKCHIP_PLL_SYNC_RATE)) - return; + return 0; drate = clk_hw_get_rate(hw); rate = rockchip_get_pll_settings(pll, drate); /* when no rate setting for the current rate, rely on clk_set_rate */ if (!rate) - return; + return 0; rockchip_rk3399_pll_get_params(pll, &cur); @@ -798,13 +802,15 @@ static void rockchip_rk3399_pll_init(struct clk_hw *hw) if (!parent) { pr_warn("%s: parent of %s not available\n", __func__, __clk_get_name(hw->clk)); - return; + return 0; } pr_debug("%s: pll %s: rate params do not match rate table, adjusting\n", __func__, __clk_get_name(hw->clk)); rockchip_rk3399_pll_set_params(pll, rate); } + + return 0; } static const struct clk_ops rockchip_rk3399_pll_clk_norate_ops = { diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c index 49bd7a4c015c..5f66bf879772 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c @@ -921,11 +921,26 @@ static const struct sunxi_ccu_desc sun50i_a64_ccu_desc = { .num_resets = ARRAY_SIZE(sun50i_a64_ccu_resets), }; +static struct ccu_pll_nb sun50i_a64_pll_cpu_nb = { + .common = &pll_cpux_clk.common, + /* copy from pll_cpux_clk */ + .enable = BIT(31), + .lock = BIT(28), +}; + +static struct ccu_mux_nb sun50i_a64_cpu_nb = { + .common = &cpux_clk.common, + .cm = &cpux_clk.mux, + .delay_us = 1, /* > 8 clock cycles at 24 MHz */ + .bypass_index = 1, /* index of 24 MHz oscillator */ +}; + static int sun50i_a64_ccu_probe(struct platform_device *pdev) { struct resource *res; void __iomem *reg; u32 val; + int ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); reg = devm_ioremap_resource(&pdev->dev, res); @@ -939,7 +954,18 @@ static int sun50i_a64_ccu_probe(struct platform_device *pdev) writel(0x515, reg + SUN50I_A64_PLL_MIPI_REG); - return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a64_ccu_desc); + ret = sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a64_ccu_desc); + if (ret) + return ret; + + /* Gate then ungate PLL CPU after any rate changes */ + ccu_pll_notifier_register(&sun50i_a64_pll_cpu_nb); + + /* Reparent CPU during PLL CPU rate changes */ + ccu_mux_notifier_register(pll_cpux_clk.common.hw.clk, + &sun50i_a64_cpu_nb); + + return 0; } static const struct of_device_id sun50i_a64_ccu_ids[] = { diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.h b/drivers/clk/sunxi-ng/ccu-sun50i-a64.h index 979929276709..116e6f826d04 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.h +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.h @@ -36,7 +36,6 @@ #define CLK_PLL_HSIC 18 #define CLK_PLL_DE 19 #define CLK_PLL_DDR1 20 -#define CLK_CPUX 21 #define CLK_AXI 22 #define CLK_APB 23 #define CLK_AHB1 24 diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.h b/drivers/clk/sunxi-ng/ccu-sun6i-a31.h index a361388b4670..3ed2a59b0dc6 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.h +++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.h @@ -32,7 +32,9 @@ /* The PLL_VIDEO1_2X clock is exported */ #define CLK_PLL_GPU 14 -#define CLK_PLL_MIPI 15 + +/* The PLL_VIDEO1_2X clock is exported */ + #define CLK_PLL9 16 #define CLK_PLL10 17 diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23-a33.h b/drivers/clk/sunxi-ng/ccu-sun8i-a23-a33.h index 72df69291cc6..5bf5c4d13b4c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a23-a33.h +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23-a33.h @@ -24,7 +24,9 @@ #define CLK_PLL_PERIPH 10 #define CLK_PLL_PERIPH_2X 11 #define CLK_PLL_GPU 12 -#define CLK_PLL_MIPI 13 + +/* The PLL MIPI clock is exported */ + #define CLK_PLL_HSIC 14 #define CLK_PLL_DE 15 #define CLK_PLL_DDR1 16 diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.h b/drivers/clk/sunxi-ng/ccu-sun8i-r40.h index a69637b6b0c1..6f7071df8e1c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.h +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.h @@ -55,10 +55,6 @@ /* Some more module clocks are exported */ -#define CLK_MBUS 155 - -/* Another bunch of module clocks are exported */ - #define CLK_NUMBER (CLK_OUTB + 1) #endif /* _CCU_SUN8I_R40_H_ */ diff --git a/drivers/clk/sunxi/clk-sun6i-apb0-gates.c b/drivers/clk/sunxi/clk-sun6i-apb0-gates.c index a165e7172346..4c75b0770c74 100644 --- a/drivers/clk/sunxi/clk-sun6i-apb0-gates.c +++ b/drivers/clk/sunxi/clk-sun6i-apb0-gates.c @@ -37,7 +37,6 @@ static int sun6i_a31_apb0_gates_clk_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct clk_onecell_data *clk_data; - const struct of_device_id *device; const struct gates_data *data; const char *clk_parent; const char *clk_name; @@ -50,10 +49,9 @@ static int sun6i_a31_apb0_gates_clk_probe(struct platform_device *pdev) if (!np) return -ENODEV; - device = of_match_device(sun6i_a31_apb0_gates_clk_dt_ids, &pdev->dev); - if (!device) + data = of_device_get_match_data(&pdev->dev); + if (!data) return -ENODEV; - data = device->data; r = platform_get_resource(pdev, IORESOURCE_MEM, 0); reg = devm_ioremap_resource(&pdev->dev, r); diff --git a/drivers/clk/tegra/clk-dfll.c b/drivers/clk/tegra/clk-dfll.c index c051d92c2bbf..cfbaa90c7adb 100644 --- a/drivers/clk/tegra/clk-dfll.c +++ b/drivers/clk/tegra/clk-dfll.c @@ -1487,7 +1487,6 @@ static int dfll_init(struct tegra_dfll *td) td->last_unrounded_rate = 0; pm_runtime_enable(td->dev); - pm_runtime_irq_safe(td->dev); pm_runtime_get_sync(td->dev); dfll_set_mode(td, DFLL_DISABLED); @@ -1516,7 +1515,7 @@ di_err1: /** * tegra_dfll_suspend - check DFLL is disabled - * @dev: DFLL device * + * @dev: DFLL instance * * DFLL clock should be disabled by the CPUFreq driver. So, make * sure it is disabled and disable all clocks needed by the DFLL. diff --git a/drivers/clk/tegra/clk-divider.c b/drivers/clk/tegra/clk-divider.c index ca0de5f11f84..38daf483ddf1 100644 --- a/drivers/clk/tegra/clk-divider.c +++ b/drivers/clk/tegra/clk-divider.c @@ -40,8 +40,13 @@ static unsigned long clk_frac_div_recalc_rate(struct clk_hw *hw, int div, mul; u64 rate = parent_rate; - reg = readl_relaxed(divider->reg) >> divider->shift; - div = reg & div_mask(divider); + reg = readl_relaxed(divider->reg); + + if ((divider->flags & TEGRA_DIVIDER_UART) && + !(reg & PERIPH_CLK_UART_DIV_ENB)) + return rate; + + div = (reg >> divider->shift) & div_mask(divider); mul = get_mul(divider); div += mul; diff --git a/drivers/clk/tegra/clk-tegra-periph.c b/drivers/clk/tegra/clk-tegra-periph.c index 0d07c0ba49b6..2b2a3b81c16b 100644 --- a/drivers/clk/tegra/clk-tegra-periph.c +++ b/drivers/clk/tegra/clk-tegra-periph.c @@ -777,7 +777,11 @@ static struct tegra_periph_init_data gate_clks[] = { GATE("ahbdma", "hclk", 33, 0, tegra_clk_ahbdma, 0), GATE("apbdma", "pclk", 34, 0, tegra_clk_apbdma, 0), GATE("kbc", "clk_32k", 36, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_kbc, 0), - GATE("fuse", "clk_m", 39, TEGRA_PERIPH_ON_APB, tegra_clk_fuse, 0), + /* + * Critical for RAM re-repair operation, which must occur on resume + * from LP1 system suspend and as part of CCPLEX cluster switching. + */ + GATE("fuse", "clk_m", 39, TEGRA_PERIPH_ON_APB, tegra_clk_fuse, CLK_IS_CRITICAL), GATE("fuse_burn", "clk_m", 39, TEGRA_PERIPH_ON_APB, tegra_clk_fuse_burn, 0), GATE("kfuse", "clk_m", 40, TEGRA_PERIPH_ON_APB, tegra_clk_kfuse, 0), GATE("apbif", "clk_m", 107, TEGRA_PERIPH_ON_APB, tegra_clk_apbif, 0), diff --git a/drivers/clk/tegra/clk-tegra20.c b/drivers/clk/tegra/clk-tegra20.c index 4d8222f5c638..fff5cba87637 100644 --- a/drivers/clk/tegra/clk-tegra20.c +++ b/drivers/clk/tegra/clk-tegra20.c @@ -1046,11 +1046,9 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA20_CLK_SBC3, TEGRA20_CLK_PLL_P, 100000000, 0 }, { TEGRA20_CLK_SBC4, TEGRA20_CLK_PLL_P, 100000000, 0 }, { TEGRA20_CLK_HOST1X, TEGRA20_CLK_PLL_C, 150000000, 0 }, - { TEGRA20_CLK_DISP1, TEGRA20_CLK_PLL_P, 600000000, 0 }, - { TEGRA20_CLK_DISP2, TEGRA20_CLK_PLL_P, 600000000, 0 }, { TEGRA20_CLK_GR2D, TEGRA20_CLK_PLL_C, 300000000, 0 }, { TEGRA20_CLK_GR3D, TEGRA20_CLK_PLL_C, 300000000, 0 }, - { TEGRA20_CLK_VDE, TEGRA20_CLK_CLK_MAX, 300000000, 0 }, + { TEGRA20_CLK_VDE, TEGRA20_CLK_PLL_C, 300000000, 0 }, /* must be the last entry */ { TEGRA20_CLK_CLK_MAX, TEGRA20_CLK_CLK_MAX, 0, 0 }, }; diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c index c8bc18e4d7e5..b20891489e11 100644 --- a/drivers/clk/tegra/clk-tegra30.c +++ b/drivers/clk/tegra/clk-tegra30.c @@ -1251,14 +1251,12 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA30_CLK_SBC6, TEGRA30_CLK_PLL_P, 100000000, 0 }, { TEGRA30_CLK_PLL_C, TEGRA30_CLK_CLK_MAX, 600000000, 0 }, { TEGRA30_CLK_HOST1X, TEGRA30_CLK_PLL_C, 150000000, 0 }, - { TEGRA30_CLK_DISP1, TEGRA30_CLK_PLL_P, 600000000, 0 }, - { TEGRA30_CLK_DISP2, TEGRA30_CLK_PLL_P, 600000000, 0 }, { TEGRA30_CLK_TWD, TEGRA30_CLK_CLK_MAX, 0, 1 }, { TEGRA30_CLK_GR2D, TEGRA30_CLK_PLL_C, 300000000, 0 }, { TEGRA30_CLK_GR3D, TEGRA30_CLK_PLL_C, 300000000, 0 }, { TEGRA30_CLK_GR3D2, TEGRA30_CLK_PLL_C, 300000000, 0 }, { TEGRA30_CLK_PLL_U, TEGRA30_CLK_CLK_MAX, 480000000, 0 }, - { TEGRA30_CLK_VDE, TEGRA30_CLK_CLK_MAX, 600000000, 0 }, + { TEGRA30_CLK_VDE, TEGRA30_CLK_PLL_C, 600000000, 0 }, { TEGRA30_CLK_SPDIF_IN_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 }, { TEGRA30_CLK_I2S0_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 }, { TEGRA30_CLK_I2S1_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 }, diff --git a/drivers/clk/ti/clk-54xx.c b/drivers/clk/ti/clk-54xx.c index c9608e5d993a..14d98a890c02 100644 --- a/drivers/clk/ti/clk-54xx.c +++ b/drivers/clk/ti/clk-54xx.c @@ -35,6 +35,20 @@ static const struct omap_clkctrl_reg_data omap5_dsp_clkctrl_regs[] __initconst = { 0 }, }; +static const char * const omap5_aess_fclk_parents[] __initconst = { + "abe_clk", + NULL, +}; + +static const struct omap_clkctrl_div_data omap5_aess_fclk_data __initconst = { + .max_div = 2, +}; + +static const struct omap_clkctrl_bit_data omap5_aess_bit_data[] __initconst = { + { 24, TI_CLK_DIVIDER, omap5_aess_fclk_parents, &omap5_aess_fclk_data }, + { 0 }, +}; + static const char * const omap5_dmic_gfclk_parents[] __initconst = { "abe_cm:clk:0018:26", "pad_clks_ck", @@ -122,6 +136,7 @@ static const struct omap_clkctrl_bit_data omap5_timer8_bit_data[] __initconst = static const struct omap_clkctrl_reg_data omap5_abe_clkctrl_regs[] __initconst = { { OMAP5_L4_ABE_CLKCTRL, NULL, 0, "abe_iclk" }, + { OMAP5_AESS_CLKCTRL, omap5_aess_bit_data, CLKF_SW_SUP, "abe_cm:clk:0008:24" }, { OMAP5_MCPDM_CLKCTRL, NULL, CLKF_SW_SUP, "pad_clks_ck" }, { OMAP5_DMIC_CLKCTRL, omap5_dmic_bit_data, CLKF_SW_SUP, "abe_cm:clk:0018:24" }, { OMAP5_MCBSP1_CLKCTRL, omap5_mcbsp1_bit_data, CLKF_SW_SUP, "abe_cm:clk:0028:24" }, diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c index 5f46782cebeb..14b645093107 100644 --- a/drivers/clk/ti/clk-7xx.c +++ b/drivers/clk/ti/clk-7xx.c @@ -146,6 +146,29 @@ static const struct omap_clkctrl_reg_data dra7_rtc_clkctrl_regs[] __initconst = { 0 }, }; +static const char * const dra7_cam_gfclk_mux_parents[] __initconst = { + "l3_iclk_div", + "core_iss_main_clk", + NULL, +}; + +static const struct omap_clkctrl_bit_data dra7_cam_bit_data[] __initconst = { + { 24, TI_CLK_MUX, dra7_cam_gfclk_mux_parents, NULL }, + { 0 }, +}; + +static const struct omap_clkctrl_reg_data dra7_cam_clkctrl_regs[] __initconst = { + { DRA7_CAM_VIP1_CLKCTRL, dra7_cam_bit_data, CLKF_HW_SUP, "l3_iclk_div" }, + { DRA7_CAM_VIP2_CLKCTRL, dra7_cam_bit_data, CLKF_HW_SUP, "l3_iclk_div" }, + { DRA7_CAM_VIP3_CLKCTRL, dra7_cam_bit_data, CLKF_HW_SUP, "l3_iclk_div" }, + { 0 }, +}; + +static const struct omap_clkctrl_reg_data dra7_vpe_clkctrl_regs[] __initconst = { + { DRA7_VPE_VPE_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h23x2_ck" }, + { 0 }, +}; + static const struct omap_clkctrl_reg_data dra7_coreaon_clkctrl_regs[] __initconst = { { DRA7_COREAON_SMARTREFLEX_MPU_CLKCTRL, NULL, CLKF_SW_SUP, "wkupaon_iclk_mux" }, { DRA7_COREAON_SMARTREFLEX_CORE_CLKCTRL, NULL, CLKF_SW_SUP, "wkupaon_iclk_mux" }, @@ -275,6 +298,40 @@ static const struct omap_clkctrl_reg_data dra7_dss_clkctrl_regs[] __initconst = { 0 }, }; +static const char * const dra7_gpu_core_mux_parents[] __initconst = { + "dpll_core_h14x2_ck", + "dpll_per_h14x2_ck", + "dpll_gpu_m2_ck", + NULL, +}; + +static const char * const dra7_gpu_hyd_mux_parents[] __initconst = { + "dpll_core_h14x2_ck", + "dpll_per_h14x2_ck", + "dpll_gpu_m2_ck", + NULL, +}; + +static const char * const dra7_gpu_sys_clk_parents[] __initconst = { + "sys_clkin", + NULL, +}; + +static const struct omap_clkctrl_div_data dra7_gpu_sys_clk_data __initconst = { + .max_div = 2, +}; + +static const struct omap_clkctrl_bit_data dra7_gpu_core_bit_data[] __initconst = { + { 24, TI_CLK_MUX, dra7_gpu_core_mux_parents, NULL, }, + { 26, TI_CLK_MUX, dra7_gpu_hyd_mux_parents, NULL, }, + { 0 }, +}; + +static const struct omap_clkctrl_reg_data dra7_gpu_clkctrl_regs[] __initconst = { + { DRA7_GPU_CLKCTRL, dra7_gpu_core_bit_data, CLKF_SW_SUP, "gpu_cm:clk:0000:24", }, + { 0 }, +}; + static const char * const dra7_mmc1_fclk_mux_parents[] __initconst = { "func_128m_clk", "dpll_per_m2x2_ck", @@ -405,7 +462,7 @@ static const struct omap_clkctrl_bit_data dra7_gmac_bit_data[] __initconst = { }; static const struct omap_clkctrl_reg_data dra7_gmac_clkctrl_regs[] __initconst = { - { DRA7_GMAC_GMAC_CLKCTRL, dra7_gmac_bit_data, CLKF_SW_SUP, "dpll_gmac_ck" }, + { DRA7_GMAC_GMAC_CLKCTRL, dra7_gmac_bit_data, CLKF_SW_SUP, "gmac_main_clk" }, { 0 }, }; @@ -769,6 +826,7 @@ const struct omap_clkctrl_data dra7_clkctrl_data[] __initconst = { { 0x4a005550, dra7_ipu_clkctrl_regs }, { 0x4a005620, dra7_dsp2_clkctrl_regs }, { 0x4a005720, dra7_rtc_clkctrl_regs }, + { 0x4a005760, dra7_vpe_clkctrl_regs }, { 0x4a008620, dra7_coreaon_clkctrl_regs }, { 0x4a008720, dra7_l3main1_clkctrl_regs }, { 0x4a008920, dra7_ipu2_clkctrl_regs }, @@ -777,7 +835,9 @@ const struct omap_clkctrl_data dra7_clkctrl_data[] __initconst = { { 0x4a008c00, dra7_atl_clkctrl_regs }, { 0x4a008d20, dra7_l4cfg_clkctrl_regs }, { 0x4a008e20, dra7_l3instr_clkctrl_regs }, + { 0x4a009020, dra7_cam_clkctrl_regs }, { 0x4a009120, dra7_dss_clkctrl_regs }, + { 0x4a009220, dra7_gpu_clkctrl_regs }, { 0x4a009320, dra7_l3init_clkctrl_regs }, { 0x4a0093b0, dra7_pcie_clkctrl_regs }, { 0x4a0093d0, dra7_gmac_clkctrl_regs }, diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c index e0b8ed3a1e80..3da33c786d77 100644 --- a/drivers/clk/ti/clk.c +++ b/drivers/clk/ti/clk.c @@ -171,7 +171,9 @@ void __init ti_dt_clocks_register(struct ti_dt_clk oclks[]) node = of_find_node_by_name(NULL, buf); if (num_args && compat_mode) { parent = node; - node = of_get_child_by_name(parent, "clk"); + node = of_get_child_by_name(parent, "clock"); + if (!node) + node = of_get_child_by_name(parent, "clk"); of_node_put(parent); } diff --git a/drivers/clk/ti/clkctrl.c b/drivers/clk/ti/clkctrl.c index 17b9a761242f..062266034d84 100644 --- a/drivers/clk/ti/clkctrl.c +++ b/drivers/clk/ti/clkctrl.c @@ -440,6 +440,63 @@ static void __init _clkctrl_add_provider(void *data, of_clk_add_hw_provider(np, _ti_omap4_clkctrl_xlate, data); } +/* Get clock name based on compatible string for clkctrl */ +static char * __init clkctrl_get_name(struct device_node *np) +{ + struct property *prop; + const int prefix_len = 11; + const char *compat; + char *name; + + of_property_for_each_string(np, "compatible", prop, compat) { + if (!strncmp("ti,clkctrl-", compat, prefix_len)) { + /* Two letter minimum name length for l3, l4 etc */ + if (strnlen(compat + prefix_len, 16) < 2) + continue; + name = kasprintf(GFP_KERNEL, "%s", compat + prefix_len); + if (!name) + continue; + strreplace(name, '-', '_'); + + return name; + } + } + of_node_put(np); + + return NULL; +} + +/* Get clkctrl clock base name based on clkctrl_name or dts node */ +static const char * __init clkctrl_get_clock_name(struct device_node *np, + const char *clkctrl_name, + int offset, int index, + bool legacy_naming) +{ + char *clock_name; + + /* l4per-clkctrl:1234:0 style naming based on clkctrl_name */ + if (clkctrl_name && !legacy_naming) { + clock_name = kasprintf(GFP_KERNEL, "%s-clkctrl:%04x:%d", + clkctrl_name, offset, index); + strreplace(clock_name, '_', '-'); + + return clock_name; + } + + /* l4per:1234:0 old style naming based on clkctrl_name */ + if (clkctrl_name) + return kasprintf(GFP_KERNEL, "%s_cm:clk:%04x:%d", + clkctrl_name, offset, index); + + /* l4per_cm:1234:0 old style naming based on parent node name */ + if (legacy_naming) + return kasprintf(GFP_KERNEL, "%pOFn:clk:%04x:%d", + np->parent, offset, index); + + /* l4per-clkctrl:1234:0 style naming based on node name */ + return kasprintf(GFP_KERNEL, "%pOFn:%04x:%d", np, offset, index); +} + static void __init _ti_omap4_clkctrl_setup(struct device_node *node) { struct omap_clkctrl_provider *provider; @@ -448,8 +505,10 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node) struct clk_init_data init = { NULL }; struct clk_hw_omap *hw; struct clk *clk; - struct omap_clkctrl_clk *clkctrl_clk; + struct omap_clkctrl_clk *clkctrl_clk = NULL; const __be32 *addrp; + bool legacy_naming; + char *clkctrl_name; u32 addr; int ret; char *c; @@ -537,7 +596,19 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node) provider->base = of_iomap(node, 0); - if (ti_clk_get_features()->flags & TI_CLK_CLKCTRL_COMPAT) { + legacy_naming = ti_clk_get_features()->flags & TI_CLK_CLKCTRL_COMPAT; + clkctrl_name = clkctrl_get_name(node); + if (clkctrl_name) { + provider->clkdm_name = kasprintf(GFP_KERNEL, + "%s_clkdm", clkctrl_name); + goto clkdm_found; + } + + /* + * The code below can be removed when all clkctrl nodes use domain + * specific compatible proprerty and standard clock node naming + */ + if (legacy_naming) { provider->clkdm_name = kasprintf(GFP_KERNEL, "%pOFnxxx", node->parent); if (!provider->clkdm_name) { kfree(provider); @@ -573,7 +644,7 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node) *c = '_'; c++; } - +clkdm_found: INIT_LIST_HEAD(&provider->clocks); /* Generate clocks */ @@ -612,15 +683,15 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node) init.flags = 0; if (reg_data->flags & CLKF_SET_RATE_PARENT) init.flags |= CLK_SET_RATE_PARENT; - if (ti_clk_get_features()->flags & TI_CLK_CLKCTRL_COMPAT) - init.name = kasprintf(GFP_KERNEL, "%pOFn:%pOFn:%04x:%d", - node->parent, node, - reg_data->offset, 0); - else - init.name = kasprintf(GFP_KERNEL, "%pOFn:%04x:%d", - node, reg_data->offset, 0); + + init.name = clkctrl_get_clock_name(node, clkctrl_name, + reg_data->offset, 0, + legacy_naming); + if (!init.name) + goto cleanup; + clkctrl_clk = kzalloc(sizeof(*clkctrl_clk), GFP_KERNEL); - if (!init.name || !clkctrl_clk) + if (!clkctrl_clk) goto cleanup; init.ops = &omap4_clkctrl_clk_ops; @@ -642,11 +713,14 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node) if (ret == -EPROBE_DEFER) ti_clk_retry_init(node, provider, _clkctrl_add_provider); + kfree(clkctrl_name); + return; cleanup: kfree(hw); kfree(init.name); + kfree(clkctrl_name); kfree(clkctrl_clk); } CLK_OF_DECLARE(ti_omap4_clkctrl_clock, "ti,clkctrl", diff --git a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h index e6995c04001e..f1dd62de2bfc 100644 --- a/drivers/clk/ti/clock.h +++ b/drivers/clk/ti/clock.h @@ -253,7 +253,7 @@ extern const struct clk_ops omap_gate_clk_ops; extern struct ti_clk_features ti_clk_features; -void omap2_init_clk_clkdm(struct clk_hw *hw); +int omap2_init_clk_clkdm(struct clk_hw *hw); int omap2_clkops_enable_clkdm(struct clk_hw *hw); void omap2_clkops_disable_clkdm(struct clk_hw *hw); diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c index 423a99b9f10c..ee56306f79d5 100644 --- a/drivers/clk/ti/clockdomain.c +++ b/drivers/clk/ti/clockdomain.c @@ -101,16 +101,16 @@ void omap2_clkops_disable_clkdm(struct clk_hw *hw) * * Convert a clockdomain name stored in a struct clk 'clk' into a * clockdomain pointer, and save it into the struct clk. Intended to be - * called during clk_register(). No return value. + * called during clk_register(). Returns 0 on success, -EERROR otherwise. */ -void omap2_init_clk_clkdm(struct clk_hw *hw) +int omap2_init_clk_clkdm(struct clk_hw *hw) { struct clk_hw_omap *clk = to_clk_hw_omap(hw); struct clockdomain *clkdm; const char *clk_name; if (!clk->clkdm_name) - return; + return 0; clk_name = __clk_get_name(hw->clk); @@ -123,6 +123,8 @@ void omap2_init_clk_clkdm(struct clk_hw *hw) pr_debug("clock: could not associate clk %s to clkdm %s\n", clk_name, clk->clkdm_name); } + + return 0; } static void __init of_ti_clockdomain_setup(struct device_node *node) diff --git a/drivers/clk/uniphier/clk-uniphier-peri.c b/drivers/clk/uniphier/clk-uniphier-peri.c index 9caa52944b1c..3e32db9dad81 100644 --- a/drivers/clk/uniphier/clk-uniphier-peri.c +++ b/drivers/clk/uniphier/clk-uniphier-peri.c @@ -18,8 +18,8 @@ #define UNIPHIER_PERI_CLK_FI2C(idx, ch) \ UNIPHIER_CLK_GATE("i2c" #ch, (idx), "i2c", 0x24, 24 + (ch)) -#define UNIPHIER_PERI_CLK_SCSSI(idx) \ - UNIPHIER_CLK_GATE("scssi", (idx), "spi", 0x20, 17) +#define UNIPHIER_PERI_CLK_SCSSI(idx, ch) \ + UNIPHIER_CLK_GATE("scssi" #ch, (idx), "spi", 0x20, 17 + (ch)) #define UNIPHIER_PERI_CLK_MCSSI(idx) \ UNIPHIER_CLK_GATE("mcssi", (idx), "spi", 0x24, 14) @@ -35,7 +35,7 @@ const struct uniphier_clk_data uniphier_ld4_peri_clk_data[] = { UNIPHIER_PERI_CLK_I2C(6, 2), UNIPHIER_PERI_CLK_I2C(7, 3), UNIPHIER_PERI_CLK_I2C(8, 4), - UNIPHIER_PERI_CLK_SCSSI(11), + UNIPHIER_PERI_CLK_SCSSI(11, 0), { /* sentinel */ } }; @@ -51,7 +51,10 @@ const struct uniphier_clk_data uniphier_pro4_peri_clk_data[] = { UNIPHIER_PERI_CLK_FI2C(8, 4), UNIPHIER_PERI_CLK_FI2C(9, 5), UNIPHIER_PERI_CLK_FI2C(10, 6), - UNIPHIER_PERI_CLK_SCSSI(11), - UNIPHIER_PERI_CLK_MCSSI(12), + UNIPHIER_PERI_CLK_SCSSI(11, 0), + UNIPHIER_PERI_CLK_SCSSI(12, 1), + UNIPHIER_PERI_CLK_SCSSI(13, 2), + UNIPHIER_PERI_CLK_SCSSI(14, 3), + UNIPHIER_PERI_CLK_MCSSI(15), { /* sentinel */ } }; diff --git a/drivers/clk/ux500/u8500_of_clk.c b/drivers/clk/ux500/u8500_of_clk.c index 72ed97c6662a..0aedd42fad52 100644 --- a/drivers/clk/ux500/u8500_of_clk.c +++ b/drivers/clk/ux500/u8500_of_clk.c @@ -99,8 +99,10 @@ static void u8500_clk_init(struct device_node *np) if (fw_version != NULL) { switch (fw_version->project) { case PRCMU_FW_PROJECT_U8500_C2: + case PRCMU_FW_PROJECT_U8500_MBL: case PRCMU_FW_PROJECT_U8520: case PRCMU_FW_PROJECT_U8420: + case PRCMU_FW_PROJECT_U8420_SYSCLK: sgaclk_parent = "soc0_pll"; break; default: diff --git a/drivers/clk/versatile/Kconfig b/drivers/clk/versatile/Kconfig index ac766855ba16..c2618f1477a2 100644 --- a/drivers/clk/versatile/Kconfig +++ b/drivers/clk/versatile/Kconfig @@ -9,7 +9,7 @@ config COMMON_CLK_VERSATILE COMPILE_TEST select REGMAP_MMIO ---help--- - Supports clocking on ARM Reference designs: + Supports clocking on ARM Reference designs: - Integrator/AP and Integrator/CP - RealView PB1176, EB, PB11MP and PBX - Versatile Express diff --git a/drivers/clk/zynqmp/clkc.c b/drivers/clk/zynqmp/clkc.c index a11f93ecbf34..10e89f23880b 100644 --- a/drivers/clk/zynqmp/clkc.c +++ b/drivers/clk/zynqmp/clkc.c @@ -2,7 +2,7 @@ /* * Zynq UltraScale+ MPSoC clock controller * - * Copyright (C) 2016-2018 Xilinx + * Copyright (C) 2016-2019 Xilinx * * Based on drivers/clk/zynq/clkc.c */ @@ -749,6 +749,7 @@ static int zynqmp_clock_probe(struct platform_device *pdev) static const struct of_device_id zynqmp_clock_of_match[] = { {.compatible = "xlnx,zynqmp-clk"}, + {.compatible = "xlnx,versal-clk"}, {}, }; MODULE_DEVICE_TABLE(of, zynqmp_clock_of_match); diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index d8f5b70d2709..4be2cc76aa2e 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -2,7 +2,7 @@ /* * Zynq UltraScale+ MPSoC Divider support * - * Copyright (C) 2016-2018 Xilinx + * Copyright (C) 2016-2019 Xilinx * * Adjustable divider clock implementation */ @@ -41,12 +41,30 @@ struct zynqmp_clk_divider { bool is_frac; u32 clk_id; u32 div_type; + u16 max_div; }; static inline int zynqmp_divider_get_val(unsigned long parent_rate, - unsigned long rate) + unsigned long rate, u16 flags) { - return DIV_ROUND_CLOSEST(parent_rate, rate); + int up, down; + unsigned long up_rate, down_rate; + + if (flags & CLK_DIVIDER_POWER_OF_TWO) { + up = DIV_ROUND_UP_ULL((u64)parent_rate, rate); + down = DIV_ROUND_DOWN_ULL((u64)parent_rate, rate); + + up = __roundup_pow_of_two(up); + down = __rounddown_pow_of_two(down); + + up_rate = DIV_ROUND_UP_ULL((u64)parent_rate, up); + down_rate = DIV_ROUND_UP_ULL((u64)parent_rate, down); + + return (rate - up_rate) <= (down_rate - rate) ? up : down; + + } else { + return DIV_ROUND_CLOSEST(parent_rate, rate); + } } /** @@ -78,6 +96,9 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, else value = div >> 16; + if (divider->flags & CLK_DIVIDER_POWER_OF_TWO) + value = 1 << value; + if (!value) { WARN(!(divider->flags & CLK_DIVIDER_ALLOW_ZERO), "%s: Zero divisor and CLK_DIVIDER_ALLOW_ZERO not set\n", @@ -88,6 +109,42 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, return DIV_ROUND_UP_ULL(parent_rate, value); } +static void zynqmp_get_divider2_val(struct clk_hw *hw, + unsigned long rate, + unsigned long parent_rate, + struct zynqmp_clk_divider *divider, + int *bestdiv) +{ + int div1; + int div2; + long error = LONG_MAX; + struct clk_hw *parent_hw = clk_hw_get_parent(hw); + struct zynqmp_clk_divider *pdivider = to_zynqmp_clk_divider(parent_hw); + + if (!pdivider) + return; + + *bestdiv = 1; + for (div1 = 1; div1 <= pdivider->max_div;) { + for (div2 = 1; div2 <= divider->max_div;) { + long new_error = ((parent_rate / div1) / div2) - rate; + + if (abs(new_error) < abs(error)) { + *bestdiv = div2; + error = new_error; + } + if (divider->flags & CLK_DIVIDER_POWER_OF_TWO) + div2 = div2 << 1; + else + div2++; + } + if (pdivider->flags & CLK_DIVIDER_POWER_OF_TWO) + div1 = div1 << 1; + else + div1++; + } +} + /** * zynqmp_clk_divider_round_rate() - Round rate of divider clock * @hw: handle between common and hardware-specific interfaces @@ -120,10 +177,23 @@ static long zynqmp_clk_divider_round_rate(struct clk_hw *hw, else bestdiv = bestdiv >> 16; + if (divider->flags & CLK_DIVIDER_POWER_OF_TWO) + bestdiv = 1 << bestdiv; + return DIV_ROUND_UP_ULL((u64)*prate, bestdiv); } - bestdiv = zynqmp_divider_get_val(*prate, rate); + bestdiv = zynqmp_divider_get_val(*prate, rate, divider->flags); + + /* + * In case of two divisors, compute best divider values and return + * divider2 value based on compute value. div1 will be automatically + * set to optimum based on required total divider value. + */ + if (div_type == TYPE_DIV2 && + (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT)) { + zynqmp_get_divider2_val(hw, rate, *prate, divider, &bestdiv); + } if ((clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && divider->is_frac) bestdiv = rate % *prate ? 1 : bestdiv; @@ -151,7 +221,7 @@ static int zynqmp_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, int ret; const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops(); - value = zynqmp_divider_get_val(parent_rate, rate); + value = zynqmp_divider_get_val(parent_rate, rate, divider->flags); if (div_type == TYPE_DIV1) { div = value & 0xFFFF; div |= 0xffff << 16; @@ -160,6 +230,9 @@ static int zynqmp_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, div |= value << 16; } + if (divider->flags & CLK_DIVIDER_POWER_OF_TWO) + div = __ffs(div); + ret = eemi_ops->clock_setdivider(clk_id, div); if (ret) @@ -176,6 +249,35 @@ static const struct clk_ops zynqmp_clk_divider_ops = { }; /** + * zynqmp_clk_get_max_divisor() - Get maximum supported divisor from firmware. + * @clk_id: Id of clock + * @type: Divider type + * + * Return: Maximum divisor of a clock if query data is successful + * U16_MAX in case of query data is not success + */ +u32 zynqmp_clk_get_max_divisor(u32 clk_id, u32 type) +{ + const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops(); + struct zynqmp_pm_query_data qdata = {0}; + u32 ret_payload[PAYLOAD_ARG_CNT]; + int ret; + + qdata.qid = PM_QID_CLOCK_GET_MAX_DIVISOR; + qdata.arg1 = clk_id; + qdata.arg2 = type; + ret = eemi_ops->query_data(qdata, ret_payload); + /* + * To maintain backward compatibility return maximum possible value + * (0xFFFF) if query for max divisor is not successful. + */ + if (ret) + return U16_MAX; + + return ret_payload[1]; +} + +/** * zynqmp_clk_register_divider() - Register a divider clock * @name: Name of this clock * @clk_id: Id of clock @@ -215,6 +317,12 @@ struct clk_hw *zynqmp_clk_register_divider(const char *name, div->clk_id = clk_id; div->div_type = nodes->type; + /* + * To achieve best possible rate, maximum limit of divider is required + * while computation. + */ + div->max_div = zynqmp_clk_get_max_divisor(clk_id, nodes->type); + hw = &div->hw; ret = clk_hw_register(NULL, hw); if (ret) { diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c index a541397a172c..89b599530105 100644 --- a/drivers/clk/zynqmp/pll.c +++ b/drivers/clk/zynqmp/pll.c @@ -188,10 +188,12 @@ static int zynqmp_pll_set_rate(struct clk_hw *hw, unsigned long rate, frac = (parent_rate * f) / FRAC_DIV; ret = eemi_ops->clock_setdivider(clk_id, m); - if (ret) + if (ret == -EUSERS) + WARN(1, "More than allowed devices are using the %s, which is forbidden\n", + clk_name); + else if (ret) pr_warn_once("%s() set divider failed for %s, ret = %d\n", __func__, clk_name, ret); - eemi_ops->ioctl(0, IOCTL_SET_PLL_FRAC_DATA, clk_id, f, NULL); return rate + frac; diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index f3ef4edd4de1..c3b1283b6d31 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -756,22 +756,21 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name) } mutex_unlock(&dma_list_mutex); - if (!IS_ERR_OR_NULL(chan)) - goto found; - - return ERR_PTR(-EPROBE_DEFER); + if (IS_ERR_OR_NULL(chan)) + return chan ? chan : ERR_PTR(-EPROBE_DEFER); found: - chan->slave = dev; chan->name = kasprintf(GFP_KERNEL, "dma:%s", name); if (!chan->name) - return ERR_PTR(-ENOMEM); + return chan; + chan->slave = dev; if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj, DMA_SLAVE_NAME)) - dev_err(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME); + dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME); if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name)) - dev_err(dev, "Cannot create DMA %s symlink\n", chan->name); + dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name); + return chan; } EXPORT_SYMBOL_GPL(dma_request_chan); @@ -830,13 +829,14 @@ void dma_release_channel(struct dma_chan *chan) /* drop PRIVATE cap enabled by __dma_request_channel() */ if (--chan->device->privatecnt == 0) dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask); + if (chan->slave) { + sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME); sysfs_remove_link(&chan->slave->kobj, chan->name); kfree(chan->name); chan->name = NULL; chan->slave = NULL; } - sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME); mutex_unlock(&dma_list_mutex); } EXPORT_SYMBOL_GPL(dma_release_channel); @@ -962,6 +962,9 @@ static int __dma_async_device_channel_register(struct dma_device *device, tchan = list_first_entry_or_null(&device->channels, struct dma_chan, device_node); + if (!tchan) + return -ENODEV; + if (tchan->dev) { idr_ref = tchan->dev->idr_ref; } else { diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 849c50ab939a..6d907fe150aa 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -66,7 +66,7 @@ static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) static inline bool is_idxd_wq_cdev(struct idxd_wq *wq) { - return wq->type == IDXD_WQT_USER ? true : false; + return wq->type == IDXD_WQT_USER; } static int idxd_config_bus_match(struct device *dev, diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c index e3850f04f676..157c959311ea 100644 --- a/drivers/dma/mv_xor_v2.c +++ b/drivers/dma/mv_xor_v2.c @@ -750,7 +750,7 @@ static int mv_xor_v2_probe(struct platform_device *pdev) } xor_dev->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) { + if (PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) { ret = EPROBE_DEFER; goto disable_reg_clk; } diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c index 899b803842bb..9dda2602c862 100644 --- a/drivers/firmware/efi/arm-runtime.c +++ b/drivers/firmware/efi/arm-runtime.c @@ -27,7 +27,7 @@ extern u64 efi_system_table; -#ifdef CONFIG_ARM64_PTDUMP_DEBUGFS +#if defined(CONFIG_PTDUMP_DEBUGFS) && defined(CONFIG_ARM64) #include <asm/ptdump.h> static struct ptdump_info efi_ptdump_info = { diff --git a/drivers/firmware/iscsi_ibft.c b/drivers/firmware/iscsi_ibft.c index 7e12cbdf957c..96758b71a8db 100644 --- a/drivers/firmware/iscsi_ibft.c +++ b/drivers/firmware/iscsi_ibft.c @@ -104,6 +104,7 @@ struct ibft_control { u16 tgt0_off; u16 nic1_off; u16 tgt1_off; + u16 expansion[0]; } __attribute__((__packed__)); struct ibft_initiator { @@ -235,7 +236,7 @@ static int ibft_verify_hdr(char *t, struct ibft_hdr *hdr, int id, int length) "found %d instead!\n", t, id, hdr->id); return -ENODEV; } - if (hdr->length != length) { + if (length && hdr->length != length) { printk(KERN_ERR "iBFT error: We expected the %s " \ "field header.length to have %d but " \ "found %d instead!\n", t, length, hdr->length); @@ -749,16 +750,16 @@ static int __init ibft_register_kobjects(struct acpi_table_ibft *header) control = (void *)header + sizeof(*header); end = (void *)control + control->hdr.length; eot_offset = (void *)header + header->header.length - (void *)control; - rc = ibft_verify_hdr("control", (struct ibft_hdr *)control, id_control, - sizeof(*control)); + rc = ibft_verify_hdr("control", (struct ibft_hdr *)control, id_control, 0); /* iBFT table safety checking */ rc |= ((control->hdr.index) ? -ENODEV : 0); + rc |= ((control->hdr.length < sizeof(*control)) ? -ENODEV : 0); if (rc) { printk(KERN_ERR "iBFT error: Control header is invalid!\n"); return rc; } - for (ptr = &control->initiator_off; ptr < end; ptr += sizeof(u16)) { + for (ptr = &control->initiator_off; ptr + sizeof(u16) <= end; ptr += sizeof(u16)) { offset = *(u16 *)ptr; if (offset && offset < header->header.length && offset < eot_offset) { diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index 75bdfaa08380..74d9f13d72c4 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -48,6 +48,8 @@ static int zynqmp_pm_ret_code(u32 ret_status) return -EACCES; case XST_PM_ABORT_SUSPEND: return -ECANCELED; + case XST_PM_MULT_USER: + return -EUSERS; case XST_PM_INTERNAL: case XST_PM_CONFLICT: case XST_PM_INVALID_NODE: diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index b585f2ef6dd9..b8013cf90064 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -1035,6 +1035,18 @@ config GPIO_BD70528 This driver can also be built as a module. If so, the module will be called gpio-bd70528. +config GPIO_BD71828 + tristate "ROHM BD71828 GPIO support" + depends on MFD_ROHM_BD71828 + help + Support for GPIOs on ROHM BD71828 PMIC. There are three GPIOs + available on the ROHM PMIC in total. The GPIOs are limited to + outputs only and pins must be configured to GPIO outputs by + OTP. Enable this only if you want to use these pins as outputs. + + This driver can also be built as a module. If so, the module + will be called gpio-bd71828. + config GPIO_BD9571MWV tristate "ROHM BD9571 GPIO support" depends on MFD_BD9571MWV diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 76ff2a5feaba..0b571264ddbc 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -37,6 +37,7 @@ obj-$(CONFIG_GPIO_ATH79) += gpio-ath79.o obj-$(CONFIG_GPIO_BCM_KONA) += gpio-bcm-kona.o obj-$(CONFIG_GPIO_BCM_XGS_IPROC) += gpio-xgs-iproc.o obj-$(CONFIG_GPIO_BD70528) += gpio-bd70528.o +obj-$(CONFIG_GPIO_BD71828) += gpio-bd71828.o obj-$(CONFIG_GPIO_BD9571MWV) += gpio-bd9571mwv.o obj-$(CONFIG_GPIO_BRCMSTB) += gpio-brcmstb.o obj-$(CONFIG_GPIO_BT8XX) += gpio-bt8xx.o diff --git a/drivers/gpio/gpio-bd71828.c b/drivers/gpio/gpio-bd71828.c new file mode 100644 index 000000000000..04aade9e0a4d --- /dev/null +++ b/drivers/gpio/gpio-bd71828.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2018 ROHM Semiconductors + +#include <linux/gpio/driver.h> +#include <linux/mfd/rohm-bd71828.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#define GPIO_OUT_REG(off) (BD71828_REG_GPIO_CTRL1 + (off)) +#define HALL_GPIO_OFFSET 3 + +/* + * These defines can be removed when + * "gpio: Add definition for GPIO direction" + * (9208b1e77d6e8e9776f34f46ef4079ecac9c3c25 in GPIO tree) gets merged, + */ +#ifndef GPIO_LINE_DIRECTION_IN + #define GPIO_LINE_DIRECTION_IN 1 + #define GPIO_LINE_DIRECTION_OUT 0 +#endif + +struct bd71828_gpio { + struct rohm_regmap_dev chip; + struct gpio_chip gpio; +}; + +static void bd71828_gpio_set(struct gpio_chip *chip, unsigned int offset, + int value) +{ + int ret; + struct bd71828_gpio *bdgpio = gpiochip_get_data(chip); + u8 val = (value) ? BD71828_GPIO_OUT_HI : BD71828_GPIO_OUT_LO; + + /* + * The HALL input pin can only be used as input. If this is the pin + * we are dealing with - then we are done + */ + if (offset == HALL_GPIO_OFFSET) + return; + + ret = regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + BD71828_GPIO_OUT_MASK, val); + if (ret) + dev_err(bdgpio->chip.dev, "Could not set gpio to %d\n", value); +} + +static int bd71828_gpio_get(struct gpio_chip *chip, unsigned int offset) +{ + int ret; + unsigned int val; + struct bd71828_gpio *bdgpio = gpiochip_get_data(chip); + + if (offset == HALL_GPIO_OFFSET) + ret = regmap_read(bdgpio->chip.regmap, BD71828_REG_IO_STAT, + &val); + else + ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + &val); + if (!ret) + ret = (val & BD71828_GPIO_OUT_MASK); + + return ret; +} + +static int bd71828_gpio_set_config(struct gpio_chip *chip, unsigned int offset, + unsigned long config) +{ + struct bd71828_gpio *bdgpio = gpiochip_get_data(chip); + + if (offset == HALL_GPIO_OFFSET) + return -ENOTSUPP; + + switch (pinconf_to_config_param(config)) { + case PIN_CONFIG_DRIVE_OPEN_DRAIN: + return regmap_update_bits(bdgpio->chip.regmap, + GPIO_OUT_REG(offset), + BD71828_GPIO_DRIVE_MASK, + BD71828_GPIO_OPEN_DRAIN); + case PIN_CONFIG_DRIVE_PUSH_PULL: + return regmap_update_bits(bdgpio->chip.regmap, + GPIO_OUT_REG(offset), + BD71828_GPIO_DRIVE_MASK, + BD71828_GPIO_PUSH_PULL); + default: + break; + } + return -ENOTSUPP; +} + +static int bd71828_get_direction(struct gpio_chip *chip, unsigned int offset) +{ + /* + * Pin usage is selected by OTP data. We can't read it runtime. Hence + * we trust that if the pin is not excluded by "gpio-reserved-ranges" + * the OTP configuration is set to OUT. (Other pins but HALL input pin + * on BD71828 can't really be used for general purpose input - input + * states are used for specific cases like regulator control or + * PMIC_ON_REQ. + */ + if (offset == HALL_GPIO_OFFSET) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; +} + +static int bd71828_probe(struct platform_device *pdev) +{ + struct bd71828_gpio *bdgpio; + struct rohm_regmap_dev *bd71828; + + bd71828 = dev_get_drvdata(pdev->dev.parent); + if (!bd71828) { + dev_err(&pdev->dev, "No MFD driver data\n"); + return -EINVAL; + } + + bdgpio = devm_kzalloc(&pdev->dev, sizeof(*bdgpio), + GFP_KERNEL); + if (!bdgpio) + return -ENOMEM; + + bdgpio->chip.dev = &pdev->dev; + bdgpio->gpio.parent = pdev->dev.parent; + bdgpio->gpio.label = "bd71828-gpio"; + bdgpio->gpio.owner = THIS_MODULE; + bdgpio->gpio.get_direction = bd71828_get_direction; + bdgpio->gpio.set_config = bd71828_gpio_set_config; + bdgpio->gpio.can_sleep = true; + bdgpio->gpio.get = bd71828_gpio_get; + bdgpio->gpio.set = bd71828_gpio_set; + bdgpio->gpio.base = -1; + + /* + * See if we need some implementation to mark some PINs as + * not controllable based on DT info or if core can handle + * "gpio-reserved-ranges" and exclude them from control + */ + bdgpio->gpio.ngpio = 4; + bdgpio->gpio.of_node = pdev->dev.parent->of_node; + bdgpio->chip.regmap = bd71828->regmap; + + return devm_gpiochip_add_data(&pdev->dev, &bdgpio->gpio, + bdgpio); +} + +static struct platform_driver bd71828_gpio = { + .driver = { + .name = "bd71828-gpio" + }, + .probe = bd71828_probe, +}; + +module_platform_driver(bd71828_gpio); + +MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>"); +MODULE_DESCRIPTION("BD71828 voltage regulator driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:bd71828-gpio"); diff --git a/drivers/gpio/gpiolib-devres.c b/drivers/gpio/gpiolib-devres.c index 4421be09b960..72b6001c56ef 100644 --- a/drivers/gpio/gpiolib-devres.c +++ b/drivers/gpio/gpiolib-devres.c @@ -308,7 +308,7 @@ devm_gpiod_get_array_optional(struct device *dev, const char *con_id, struct gpio_descs *descs; descs = devm_gpiod_get_array(dev, con_id, flags); - if (IS_ERR(descs) && (PTR_ERR(descs) == -ENOENT)) + if (PTR_ERR(descs) == -ENOENT) return NULL; return descs; diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 1b3f217a35e2..c6d30f73df07 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -484,24 +484,24 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, break; } - if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) { + if (PTR_ERR(desc) == -ENOENT) { /* Special handling for SPI GPIOs if used */ desc = of_find_spi_gpio(dev, con_id, &of_flags); } - if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) { + if (PTR_ERR(desc) == -ENOENT) { /* This quirk looks up flags and all */ desc = of_find_spi_cs_gpio(dev, con_id, idx, flags); if (!IS_ERR(desc)) return desc; } - if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) { + if (PTR_ERR(desc) == -ENOENT) { /* Special handling for regulator GPIOs if used */ desc = of_find_regulator_gpio(dev, con_id, &of_flags); } - if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) + if (PTR_ERR(desc) == -ENOENT) desc = of_find_arizona_gpio(dev, con_id, &of_flags); if (IS_ERR(desc)) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 99ac27a72e28..753283486037 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -5039,7 +5039,7 @@ struct gpio_descs *__must_check gpiod_get_array_optional(struct device *dev, struct gpio_descs *descs; descs = gpiod_get_array(dev, con_id, flags); - if (IS_ERR(descs) && (PTR_ERR(descs) == -ENOENT)) + if (PTR_ERR(descs) == -ENOENT) return NULL; return descs; diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 7ae3b22c5628..c2bbcdd9c875 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -120,6 +120,7 @@ amdgpu-y += \ amdgpu_rlc.o \ gfx_v8_0.o \ gfx_v9_0.o \ + gfx_v9_4.o \ gfx_v10_0.o # add async DMA block diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index b1bb10625cd9..da3bcff61b97 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1009,10 +1009,14 @@ int emu_soc_asic_init(struct amdgpu_device *adev); #define AMDGPU_REGS_IDX (1<<0) #define AMDGPU_REGS_NO_KIQ (1<<1) +#define AMDGPU_REGS_KIQ (1<<2) #define RREG32_NO_KIQ(reg) amdgpu_mm_rreg(adev, (reg), AMDGPU_REGS_NO_KIQ) #define WREG32_NO_KIQ(reg, v) amdgpu_mm_wreg(adev, (reg), (v), AMDGPU_REGS_NO_KIQ) +#define RREG32_KIQ(reg) amdgpu_mm_rreg(adev, (reg), AMDGPU_REGS_KIQ) +#define WREG32_KIQ(reg, v) amdgpu_mm_wreg(adev, (reg), (v), AMDGPU_REGS_KIQ) + #define RREG8(reg) amdgpu_mm_rreg8(adev, (reg)) #define WREG8(reg, v) amdgpu_mm_wreg8(adev, (reg), (v)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 82155ac3288a..12247a32f9ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -527,7 +527,7 @@ static int acp_set_powergating_state(void *handle, enum amd_powergating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = state == AMD_PG_STATE_GATE ? true : false; + bool enable = (state == AMD_PG_STATE_GATE); if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->set_powergating_by_smu) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index b2487f4f271b..fa8ac9d19a7a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -2129,6 +2129,7 @@ int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem return -ENOMEM; mutex_init(&(*mem)->lock); + INIT_LIST_HEAD(&(*mem)->bo_va_list); (*mem)->bo = amdgpu_bo_ref(gws_bo); (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; (*mem)->process_info = process_info; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 64e2babbc36e..94a6c42f29ea 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -42,19 +42,12 @@ const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = { [AMDGPU_HW_IP_VCN_JPEG] = 1, }; -static int amdgpu_ctx_total_num_entities(void) -{ - unsigned i, num_entities = 0; - - for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) - num_entities += amdgpu_ctx_num_entities[i]; - - return num_entities; -} - static int amdgpu_ctx_priority_permit(struct drm_file *filp, enum drm_sched_priority priority) { + if (priority < 0 || priority >= DRM_SCHED_PRIORITY_MAX) + return -EINVAL; + /* NORMAL and below are accessible by everyone */ if (priority <= DRM_SCHED_PRIORITY_NORMAL) return 0; @@ -68,64 +61,24 @@ static int amdgpu_ctx_priority_permit(struct drm_file *filp, return -EACCES; } -static int amdgpu_ctx_init(struct amdgpu_device *adev, - enum drm_sched_priority priority, - struct drm_file *filp, - struct amdgpu_ctx *ctx) +static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, const u32 hw_ip, const u32 ring) { - unsigned num_entities = amdgpu_ctx_total_num_entities(); - unsigned i, j; + struct amdgpu_device *adev = ctx->adev; + struct amdgpu_ctx_entity *entity; + struct drm_gpu_scheduler **scheds = NULL, *sched = NULL; + unsigned num_scheds = 0; + enum drm_sched_priority priority; int r; - if (priority < 0 || priority >= DRM_SCHED_PRIORITY_MAX) - return -EINVAL; - - r = amdgpu_ctx_priority_permit(filp, priority); - if (r) - return r; + entity = kcalloc(1, offsetof(typeof(*entity), fences[amdgpu_sched_jobs]), + GFP_KERNEL); + if (!entity) + return -ENOMEM; - memset(ctx, 0, sizeof(*ctx)); - ctx->adev = adev; - - ctx->fences = kcalloc(amdgpu_sched_jobs * num_entities, - sizeof(struct dma_fence*), GFP_KERNEL); - if (!ctx->fences) - return -ENOMEM; - - ctx->entities[0] = kcalloc(num_entities, - sizeof(struct amdgpu_ctx_entity), - GFP_KERNEL); - if (!ctx->entities[0]) { - r = -ENOMEM; - goto error_free_fences; - } - - for (i = 0; i < num_entities; ++i) { - struct amdgpu_ctx_entity *entity = &ctx->entities[0][i]; - - entity->sequence = 1; - entity->fences = &ctx->fences[amdgpu_sched_jobs * i]; - } - for (i = 1; i < AMDGPU_HW_IP_NUM; ++i) - ctx->entities[i] = ctx->entities[i - 1] + - amdgpu_ctx_num_entities[i - 1]; - - kref_init(&ctx->refcount); - spin_lock_init(&ctx->ring_lock); - mutex_init(&ctx->lock); - - ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); - ctx->reset_counter_query = ctx->reset_counter; - ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); - ctx->init_priority = priority; - ctx->override_priority = DRM_SCHED_PRIORITY_UNSET; - - for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { - struct drm_gpu_scheduler **scheds; - struct drm_gpu_scheduler *sched; - unsigned num_scheds = 0; - - switch (i) { + entity->sequence = 1; + priority = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ? + ctx->init_priority : ctx->override_priority; + switch (hw_ip) { case AMDGPU_HW_IP_GFX: sched = &adev->gfx.gfx_ring[0].sched; scheds = &sched; @@ -166,53 +119,90 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, scheds = adev->jpeg.jpeg_sched; num_scheds = adev->jpeg.num_jpeg_sched; break; - } - - for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) - r = drm_sched_entity_init(&ctx->entities[i][j].entity, - priority, scheds, - num_scheds, &ctx->guilty); - if (r) - goto error_cleanup_entities; } + r = drm_sched_entity_init(&entity->entity, priority, scheds, num_scheds, + &ctx->guilty); + if (r) + goto error_free_entity; + + ctx->entities[hw_ip][ring] = entity; return 0; -error_cleanup_entities: - for (i = 0; i < num_entities; ++i) - drm_sched_entity_destroy(&ctx->entities[0][i].entity); - kfree(ctx->entities[0]); +error_free_entity: + kfree(entity); -error_free_fences: - kfree(ctx->fences); - ctx->fences = NULL; return r; } +static int amdgpu_ctx_init(struct amdgpu_device *adev, + enum drm_sched_priority priority, + struct drm_file *filp, + struct amdgpu_ctx *ctx) +{ + int r; + + r = amdgpu_ctx_priority_permit(filp, priority); + if (r) + return r; + + memset(ctx, 0, sizeof(*ctx)); + + ctx->adev = adev; + + kref_init(&ctx->refcount); + spin_lock_init(&ctx->ring_lock); + mutex_init(&ctx->lock); + + ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); + ctx->reset_counter_query = ctx->reset_counter; + ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); + ctx->init_priority = priority; + ctx->override_priority = DRM_SCHED_PRIORITY_UNSET; + + return 0; + +} + +static void amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity) +{ + + int i; + + if (!entity) + return; + + for (i = 0; i < amdgpu_sched_jobs; ++i) + dma_fence_put(entity->fences[i]); + + kfree(entity); +} + static void amdgpu_ctx_fini(struct kref *ref) { struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount); - unsigned num_entities = amdgpu_ctx_total_num_entities(); struct amdgpu_device *adev = ctx->adev; unsigned i, j; if (!adev) return; - for (i = 0; i < num_entities; ++i) - for (j = 0; j < amdgpu_sched_jobs; ++j) - dma_fence_put(ctx->entities[0][i].fences[j]); - kfree(ctx->fences); - kfree(ctx->entities[0]); + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { + for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) { + amdgpu_ctx_fini_entity(ctx->entities[i][j]); + ctx->entities[i][j] = NULL; + } + } mutex_destroy(&ctx->lock); - kfree(ctx); } int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, u32 ring, struct drm_sched_entity **entity) { + int r; + if (hw_ip >= AMDGPU_HW_IP_NUM) { DRM_ERROR("unknown HW IP type: %d\n", hw_ip); return -EINVAL; @@ -229,7 +219,13 @@ int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, return -EINVAL; } - *entity = &ctx->entities[hw_ip][ring].entity; + if (ctx->entities[hw_ip][ring] == NULL) { + r = amdgpu_ctx_init_entity(ctx, hw_ip, ring); + if (r) + return r; + } + + *entity = &ctx->entities[hw_ip][ring]->entity; return 0; } @@ -269,14 +265,17 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev, static void amdgpu_ctx_do_release(struct kref *ref) { struct amdgpu_ctx *ctx; - unsigned num_entities; - u32 i; + u32 i, j; ctx = container_of(ref, struct amdgpu_ctx, refcount); + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { + for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { + if (!ctx->entities[i][j]) + continue; - num_entities = amdgpu_ctx_total_num_entities(); - for (i = 0; i < num_entities; i++) - drm_sched_entity_destroy(&ctx->entities[0][i].entity); + drm_sched_entity_destroy(&ctx->entities[i][j]->entity); + } + } amdgpu_ctx_fini(ref); } @@ -506,19 +505,23 @@ struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, enum drm_sched_priority priority) { - unsigned num_entities = amdgpu_ctx_total_num_entities(); enum drm_sched_priority ctx_prio; - unsigned i; + unsigned i, j; ctx->override_priority = priority; ctx_prio = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ? ctx->init_priority : ctx->override_priority; + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { + for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { + struct drm_sched_entity *entity; - for (i = 0; i < num_entities; i++) { - struct drm_sched_entity *entity = &ctx->entities[0][i].entity; + if (!ctx->entities[i][j]) + continue; - drm_sched_entity_set_priority(entity, ctx_prio); + entity = &ctx->entities[i][j]->entity; + drm_sched_entity_set_priority(entity, ctx_prio); + } } } @@ -554,20 +557,24 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr) long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) { - unsigned num_entities = amdgpu_ctx_total_num_entities(); struct amdgpu_ctx *ctx; struct idr *idp; - uint32_t id, i; + uint32_t id, i, j; idp = &mgr->ctx_handles; mutex_lock(&mgr->lock); idr_for_each_entry(idp, ctx, id) { - for (i = 0; i < num_entities; i++) { - struct drm_sched_entity *entity; + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { + for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { + struct drm_sched_entity *entity; + + if (!ctx->entities[i][j]) + continue; - entity = &ctx->entities[0][i].entity; - timeout = drm_sched_entity_flush(entity, timeout); + entity = &ctx->entities[i][j]->entity; + timeout = drm_sched_entity_flush(entity, timeout); + } } } mutex_unlock(&mgr->lock); @@ -576,10 +583,9 @@ long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) { - unsigned num_entities = amdgpu_ctx_total_num_entities(); struct amdgpu_ctx *ctx; struct idr *idp; - uint32_t id, i; + uint32_t id, i, j; idp = &mgr->ctx_handles; @@ -589,8 +595,17 @@ void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) continue; } - for (i = 0; i < num_entities; i++) - drm_sched_entity_fini(&ctx->entities[0][i].entity); + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { + for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { + struct drm_sched_entity *entity; + + if (!ctx->entities[i][j]) + continue; + + entity = &ctx->entities[i][j]->entity; + drm_sched_entity_fini(entity); + } + } } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h index 4ad90a44dc3c..de490f183af2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h @@ -29,10 +29,12 @@ struct drm_device; struct drm_file; struct amdgpu_fpriv; +#define AMDGPU_MAX_ENTITY_NUM 4 + struct amdgpu_ctx_entity { uint64_t sequence; - struct dma_fence **fences; struct drm_sched_entity entity; + struct dma_fence *fences[]; }; struct amdgpu_ctx { @@ -42,8 +44,7 @@ struct amdgpu_ctx { unsigned reset_counter_query; uint32_t vram_lost_counter; spinlock_t ring_lock; - struct dma_fence **fences; - struct amdgpu_ctx_entity *entities[AMDGPU_HW_IP_NUM]; + struct amdgpu_ctx_entity *entities[AMDGPU_HW_IP_NUM][AMDGPU_MAX_ENTITY_NUM]; bool preamble_presented; enum drm_sched_priority init_priority; enum drm_sched_priority override_priority; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 53d882000101..39cd545976b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -216,8 +216,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, { uint32_t ret; - if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) - return amdgpu_virt_kiq_rreg(adev, reg); + if ((acc_flags & AMDGPU_REGS_KIQ) || (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))) + return amdgpu_kiq_rreg(adev, reg); if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -294,8 +294,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v; } - if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) - return amdgpu_virt_kiq_wreg(adev, reg, v); + if ((acc_flags & AMDGPU_REGS_KIQ) || (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))) + return amdgpu_kiq_wreg(adev, reg, v); if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); @@ -985,7 +985,7 @@ static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) { struct sysinfo si; - bool is_os_64 = (sizeof(void *) == 8) ? true : false; + bool is_os_64 = (sizeof(void *) == 8); uint64_t total_memory; uint64_t dram_size_seven_GB = 0x1B8000000; uint64_t dram_size_three_GB = 0xB8000000; @@ -3760,6 +3760,10 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) case CHIP_VEGA12: case CHIP_RAVEN: case CHIP_ARCTURUS: + case CHIP_RENOIR: + case CHIP_NAVI10: + case CHIP_NAVI14: + case CHIP_NAVI12: break; default: goto disabled; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_df.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_df.h index 61a26c15c8dd..057f6ea645d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_df.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_df.h @@ -52,6 +52,9 @@ struct amdgpu_df_funcs { uint64_t (*get_fica)(struct amdgpu_device *adev, uint32_t ficaa_val); void (*set_fica)(struct amdgpu_device *adev, uint32_t ficaa_val, uint32_t ficadl_val, uint32_t ficadh_val); + uint64_t (*get_dram_base_addr)(struct amdgpu_device *adev, + uint32_t df_inst); + uint32_t (*get_df_inst_id)(struct amdgpu_device *adev); }; struct amdgpu_df { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index b88b8b82bb64..0f960b498792 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -296,7 +296,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, spin_lock_init(&kiq->ring_lock); - r = amdgpu_device_wb_get(adev, &adev->virt.reg_val_offs); + r = amdgpu_device_wb_get(adev, &kiq->reg_val_offs); if (r) return r; @@ -321,7 +321,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring) { - amdgpu_device_wb_free(ring->adev, ring->adev->virt.reg_val_offs); + amdgpu_device_wb_free(ring->adev, ring->adev->gfx.kiq.reg_val_offs); amdgpu_ring_fini(ring); } @@ -658,3 +658,95 @@ int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, amdgpu_ras_interrupt_dispatch(adev, &ih_data); return 0; } + +uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) +{ + signed long r, cnt = 0; + unsigned long flags; + uint32_t seq; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + struct amdgpu_ring *ring = &kiq->ring; + + BUG_ON(!ring->funcs->emit_rreg); + + spin_lock_irqsave(&kiq->ring_lock, flags); + amdgpu_ring_alloc(ring, 32); + amdgpu_ring_emit_rreg(ring, reg); + amdgpu_fence_emit_polling(ring, &seq); + amdgpu_ring_commit(ring); + spin_unlock_irqrestore(&kiq->ring_lock, flags); + + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + + /* don't wait anymore for gpu reset case because this way may + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will + * never return if we keep waiting in virt_kiq_rreg, which cause + * gpu_recover() hang there. + * + * also don't wait anymore for IRQ context + * */ + if (r < 1 && (adev->in_gpu_reset || in_interrupt())) + goto failed_kiq_read; + + might_sleep(); + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + } + + if (cnt > MAX_KIQ_REG_TRY) + goto failed_kiq_read; + + return adev->wb.wb[kiq->reg_val_offs]; + +failed_kiq_read: + pr_err("failed to read reg:%x\n", reg); + return ~0; +} + +void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) +{ + signed long r, cnt = 0; + unsigned long flags; + uint32_t seq; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + struct amdgpu_ring *ring = &kiq->ring; + + BUG_ON(!ring->funcs->emit_wreg); + + spin_lock_irqsave(&kiq->ring_lock, flags); + amdgpu_ring_alloc(ring, 32); + amdgpu_ring_emit_wreg(ring, reg, v); + amdgpu_fence_emit_polling(ring, &seq); + amdgpu_ring_commit(ring); + spin_unlock_irqrestore(&kiq->ring_lock, flags); + + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + + /* don't wait anymore for gpu reset case because this way may + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will + * never return if we keep waiting in virt_kiq_rreg, which cause + * gpu_recover() hang there. + * + * also don't wait anymore for IRQ context + * */ + if (r < 1 && (adev->in_gpu_reset || in_interrupt())) + goto failed_kiq_write; + + might_sleep(); + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { + + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + } + + if (cnt > MAX_KIQ_REG_TRY) + goto failed_kiq_write; + + return; + +failed_kiq_write: + pr_err("failed to write reg:%x\n", reg); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index af4bd279f42f..ca17ffb01301 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -94,6 +94,7 @@ struct amdgpu_kiq { struct amdgpu_ring ring; struct amdgpu_irq_src irq; const struct kiq_pm4_funcs *pmf; + uint32_t reg_val_offs; }; /* @@ -375,4 +376,6 @@ int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev, int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry); +uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); +void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h index 86267baca07c..d3c27a3c43f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h @@ -60,11 +60,6 @@ */ #define AMDGPU_GMC_FAULT_TIMEOUT 5000ULL -/* - * Default stolen memory size, 1024 * 768 * 4 - */ -#define AMDGPU_STOLEN_BIST_TRAINING_DEFAULT_SIZE 0x300000ULL - struct firmware; /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 3265487b859f..611021514c52 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -172,6 +172,8 @@ struct psp_dtm_context { #define MEM_TRAIN_SYSTEM_SIGNATURE 0x54534942 #define GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES 0x1000 #define GDDR6_MEM_TRAINING_OFFSET 0x8000 +/*Define the VRAM size that will be encroached by BIST training.*/ +#define GDDR6_MEM_TRAINING_ENCROACHED_SIZE 0x2000000 enum psp_memory_training_init_flag { PSP_MEM_TRAIN_NOT_SUPPORT = 0x0, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 766be7f18282..cef94e2169fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -742,6 +742,20 @@ int amdgpu_ras_error_query(struct amdgpu_device *adev, return 0; } +uint64_t get_xgmi_relative_phy_addr(struct amdgpu_device *adev, uint64_t addr) +{ + uint32_t df_inst_id; + + if ((!adev->df.funcs) || + (!adev->df.funcs->get_df_inst_id) || + (!adev->df.funcs->get_dram_base_addr)) + return addr; + + df_inst_id = adev->df.funcs->get_df_inst_id(adev); + + return addr + adev->df.funcs->get_dram_base_addr(adev, df_inst_id); +} + /* wrapper of psp_ras_trigger_error */ int amdgpu_ras_error_inject(struct amdgpu_device *adev, struct ras_inject_if *info) @@ -759,6 +773,12 @@ int amdgpu_ras_error_inject(struct amdgpu_device *adev, if (!obj) return -EINVAL; + /* Calculate XGMI relative offset */ + if (adev->gmc.xgmi.num_physical_nodes > 1) { + block_info.address = get_xgmi_relative_phy_addr(adev, + block_info.address); + } + switch (info->head.block) { case AMDGPU_RAS_BLOCK__GFX: if (adev->gfx.funcs->ras_error_inject) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index ceb0dbf685f1..59ddba137946 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -652,7 +652,7 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx, if ((addr + (uint64_t)size) > (mapping->last + 1) * AMDGPU_GPU_PAGE_SIZE) { - DRM_ERROR("BO to small for addr 0x%010Lx %d %d\n", + DRM_ERROR("BO too small for addr 0x%010Lx %d %d\n", addr, lo, hi); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h index c4984c5fb2db..d6deb0eb1e15 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h @@ -65,33 +65,33 @@ /* 1 second timeout */ #define VCN_IDLE_TIMEOUT msecs_to_jiffies(1000) -#define RREG32_SOC15_DPG_MODE(ip, inst, reg, mask, sram_sel) \ - ({ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \ - WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \ +#define RREG32_SOC15_DPG_MODE(ip, inst_idx, reg, mask, sram_sel) \ + ({ WREG32_SOC15(ip, inst_idx, mmUVD_DPG_LMA_MASK, mask); \ + WREG32_SOC15(ip, inst_idx, mmUVD_DPG_LMA_CTL, \ UVD_DPG_LMA_CTL__MASK_EN_MASK | \ - ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \ + ((adev->reg_offset[ip##_HWIP][inst_idx][reg##_BASE_IDX] + reg) \ << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \ (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \ - RREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA); \ + RREG32_SOC15(ip, inst_idx, mmUVD_DPG_LMA_DATA); \ }) -#define WREG32_SOC15_DPG_MODE(ip, inst, reg, value, mask, sram_sel) \ +#define WREG32_SOC15_DPG_MODE(ip, inst_idx, reg, value, mask, sram_sel) \ do { \ - WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA, value); \ - WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \ - WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \ + WREG32_SOC15(ip, inst_idx, mmUVD_DPG_LMA_DATA, value); \ + WREG32_SOC15(ip, inst_idx, mmUVD_DPG_LMA_MASK, mask); \ + WREG32_SOC15(ip, inst_idx, mmUVD_DPG_LMA_CTL, \ UVD_DPG_LMA_CTL__READ_WRITE_MASK | \ - ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \ + ((adev->reg_offset[ip##_HWIP][inst_idx][reg##_BASE_IDX] + reg) \ << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \ (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \ } while (0) -#define SOC15_DPG_MODE_OFFSET_2_0(ip, inst, reg) \ +#define SOC15_DPG_MODE_OFFSET_2_0(ip, inst_idx, reg) \ ({ \ uint32_t internal_reg_offset, addr; \ bool video_range, aon_range; \ \ - addr = (adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg); \ + addr = (adev->reg_offset[ip##_HWIP][inst_idx][reg##_BASE_IDX] + reg); \ addr <<= 2; \ video_range = ((((0xFFFFF & addr) >= (VCN_VID_SOC_ADDRESS_2_0)) && \ ((0xFFFFF & addr) < ((VCN_VID_SOC_ADDRESS_2_0 + 0x2600))))); \ @@ -111,7 +111,7 @@ #define RREG32_SOC15_DPG_MODE_2_0(inst_idx, offset, mask_en) \ ({ \ - WREG32_SOC15(VCN, inst, mmUVD_DPG_LMA_CTL, \ + WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_LMA_CTL, \ (0x0 << UVD_DPG_LMA_CTL__READ_WRITE__SHIFT | \ mask_en << UVD_DPG_LMA_CTL__MASK_EN__SHIFT | \ offset << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT)); \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 103033f96f13..adc813cde8e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -45,98 +45,6 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev) adev->pg_flags = 0; } -uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) -{ - signed long r, cnt = 0; - unsigned long flags; - uint32_t seq; - struct amdgpu_kiq *kiq = &adev->gfx.kiq; - struct amdgpu_ring *ring = &kiq->ring; - - BUG_ON(!ring->funcs->emit_rreg); - - spin_lock_irqsave(&kiq->ring_lock, flags); - amdgpu_ring_alloc(ring, 32); - amdgpu_ring_emit_rreg(ring, reg); - amdgpu_fence_emit_polling(ring, &seq); - amdgpu_ring_commit(ring); - spin_unlock_irqrestore(&kiq->ring_lock, flags); - - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - - /* don't wait anymore for gpu reset case because this way may - * block gpu_recover() routine forever, e.g. this virt_kiq_rreg - * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will - * never return if we keep waiting in virt_kiq_rreg, which cause - * gpu_recover() hang there. - * - * also don't wait anymore for IRQ context - * */ - if (r < 1 && (adev->in_gpu_reset || in_interrupt())) - goto failed_kiq_read; - - might_sleep(); - while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { - msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - } - - if (cnt > MAX_KIQ_REG_TRY) - goto failed_kiq_read; - - return adev->wb.wb[adev->virt.reg_val_offs]; - -failed_kiq_read: - pr_err("failed to read reg:%x\n", reg); - return ~0; -} - -void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) -{ - signed long r, cnt = 0; - unsigned long flags; - uint32_t seq; - struct amdgpu_kiq *kiq = &adev->gfx.kiq; - struct amdgpu_ring *ring = &kiq->ring; - - BUG_ON(!ring->funcs->emit_wreg); - - spin_lock_irqsave(&kiq->ring_lock, flags); - amdgpu_ring_alloc(ring, 32); - amdgpu_ring_emit_wreg(ring, reg, v); - amdgpu_fence_emit_polling(ring, &seq); - amdgpu_ring_commit(ring); - spin_unlock_irqrestore(&kiq->ring_lock, flags); - - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - - /* don't wait anymore for gpu reset case because this way may - * block gpu_recover() routine forever, e.g. this virt_kiq_rreg - * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will - * never return if we keep waiting in virt_kiq_rreg, which cause - * gpu_recover() hang there. - * - * also don't wait anymore for IRQ context - * */ - if (r < 1 && (adev->in_gpu_reset || in_interrupt())) - goto failed_kiq_write; - - might_sleep(); - while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { - - msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); - r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); - } - - if (cnt > MAX_KIQ_REG_TRY) - goto failed_kiq_write; - - return; - -failed_kiq_write: - pr_err("failed to write reg:%x\n", reg); -} - void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h index 4d1ac7612967..daaf909d009a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h @@ -287,8 +287,6 @@ static inline bool is_virtual_machine(void) bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev); void amdgpu_virt_init_setting(struct amdgpu_device *adev); -uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); -void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v); void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, uint32_t reg0, uint32_t rreg1, uint32_t ref, uint32_t mask); diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c index d9cc746af5e6..847ca9b3ce4e 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v1_0.c @@ -74,9 +74,9 @@ int athub_v1_0_set_clockgating(struct amdgpu_device *adev, case CHIP_VEGA20: case CHIP_RAVEN: athub_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); athub_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c index ceb9aa4df0e7..921a69abda55 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c @@ -77,9 +77,9 @@ int athub_v2_0_set_clockgating(struct amdgpu_device *adev, case CHIP_NAVI14: case CHIP_NAVI12: athub_v2_0_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); athub_v2_0_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c index f51326598a8c..5a1bd8ed1a6c 100644 --- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c +++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c @@ -27,6 +27,9 @@ #include "df/df_3_6_offset.h" #include "df/df_3_6_sh_mask.h" +#define DF_3_6_SMN_REG_INST_DIST 0x8 +#define DF_3_6_INST_CNT 8 + static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 32, 0, 0, 0, 2, 4, 8}; @@ -683,6 +686,58 @@ static void df_v3_6_pmc_get_count(struct amdgpu_device *adev, } } +static uint64_t df_v3_6_get_dram_base_addr(struct amdgpu_device *adev, + uint32_t df_inst) +{ + uint32_t base_addr_reg_val = 0; + uint64_t base_addr = 0; + + base_addr_reg_val = RREG32_PCIE(smnDF_CS_UMC_AON0_DramBaseAddress0 + + df_inst * DF_3_6_SMN_REG_INST_DIST); + + if (REG_GET_FIELD(base_addr_reg_val, + DF_CS_UMC_AON0_DramBaseAddress0, + AddrRngVal) == 0) { + DRM_WARN("address range not valid"); + return 0; + } + + base_addr = REG_GET_FIELD(base_addr_reg_val, + DF_CS_UMC_AON0_DramBaseAddress0, + DramBaseAddr); + + return base_addr << 28; +} + +static uint32_t df_v3_6_get_df_inst_id(struct amdgpu_device *adev) +{ + uint32_t xgmi_node_id = 0; + uint32_t df_inst_id = 0; + + /* Walk through DF dst nodes to find current XGMI node */ + for (df_inst_id = 0; df_inst_id < DF_3_6_INST_CNT; df_inst_id++) { + + xgmi_node_id = RREG32_PCIE(smnDF_CS_UMC_AON0_DramLimitAddress0 + + df_inst_id * DF_3_6_SMN_REG_INST_DIST); + xgmi_node_id = REG_GET_FIELD(xgmi_node_id, + DF_CS_UMC_AON0_DramLimitAddress0, + DstFabricID); + + /* TODO: establish reason dest fabric id is offset by 7 */ + xgmi_node_id = xgmi_node_id >> 7; + + if (adev->gmc.xgmi.physical_node_id == xgmi_node_id) + break; + } + + if (df_inst_id == DF_3_6_INST_CNT) { + DRM_WARN("cant match df dst id with gpu node"); + return 0; + } + + return df_inst_id; +} + const struct amdgpu_df_funcs df_v3_6_funcs = { .sw_init = df_v3_6_sw_init, .sw_fini = df_v3_6_sw_fini, @@ -696,5 +751,7 @@ const struct amdgpu_df_funcs df_v3_6_funcs = { .pmc_stop = df_v3_6_pmc_stop, .pmc_get_count = df_v3_6_pmc_get_count, .get_fica = df_v3_6_get_fica, - .set_fica = df_v3_6_set_fica + .set_fica = df_v3_6_set_fica, + .get_dram_base_addr = df_v3_6_get_dram_base_addr, + .get_df_inst_id = df_v3_6_get_df_inst_id }; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 874f641de281..1785fdad6ecb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -368,7 +368,7 @@ static const struct kiq_pm4_funcs gfx_v10_0_kiq_pm4_funcs = { .map_queues_size = 7, .unmap_queues_size = 6, .query_status_size = 7, - .invalidate_tlbs_size = 12, + .invalidate_tlbs_size = 2, }; static void gfx_v10_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) @@ -4229,7 +4229,7 @@ static int gfx_v10_0_set_powergating_state(void *handle, enum amd_powergating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_PG_STATE_GATE) ? true : false; + bool enable = (state == AMD_PG_STATE_GATE); switch (adev->asic_type) { case CHIP_NAVI10: case CHIP_NAVI14: @@ -4255,7 +4255,7 @@ static int gfx_v10_0_set_clockgating_state(void *handle, case CHIP_NAVI14: case CHIP_NAVI12: gfx_v10_0_update_gfx_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; @@ -4737,6 +4737,7 @@ static void gfx_v10_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start) static void gfx_v10_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) { struct amdgpu_device *adev = ring->adev; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); amdgpu_ring_write(ring, 0 | /* src: register*/ @@ -4745,9 +4746,9 @@ static void gfx_v10_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) amdgpu_ring_write(ring, reg); amdgpu_ring_write(ring, 0); amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + - adev->virt.reg_val_offs * 4)); + kiq->reg_val_offs * 4)); amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + - adev->virt.reg_val_offs * 4)); + kiq->reg_val_offs * 4)); } static void gfx_v10_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 46f0533ba43f..fa245973de12 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -6449,6 +6449,7 @@ static void gfx_v8_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigne static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) { struct amdgpu_device *adev = ring->adev; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); amdgpu_ring_write(ring, 0 | /* src: register*/ @@ -6457,9 +6458,9 @@ static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) amdgpu_ring_write(ring, reg); amdgpu_ring_write(ring, 0); amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + - adev->virt.reg_val_offs * 4)); + kiq->reg_val_offs * 4)); amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + - adev->virt.reg_val_offs * 4)); + kiq->reg_val_offs * 4)); } static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 46ab46757b25..90f64b8bc358 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -48,6 +48,8 @@ #include "amdgpu_ras.h" +#include "gfx_v9_4.h" + #define GFX9_NUM_GFX_RINGS 1 #define GFX9_MEC_HPD_SIZE 4096 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L @@ -736,6 +738,7 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status); +static void gfx_v9_0_clear_ras_edc_counter(struct amdgpu_device *adev); static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, void *inject_if); @@ -859,7 +862,7 @@ static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { .map_queues_size = 7, .unmap_queues_size = 6, .query_status_size = 7, - .invalidate_tlbs_size = 12, + .invalidate_tlbs_size = 2, }; static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) @@ -1159,18 +1162,54 @@ static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) } } +struct amdgpu_gfxoff_quirk { + u16 chip_vendor; + u16 chip_device; + u16 subsys_vendor; + u16 subsys_device; + u8 revision; +}; + +static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { + /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ + { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, + { 0, 0, 0, 0, 0 }, +}; + +static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) +{ + const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; + + while (p && p->chip_device != 0) { + if (pdev->vendor == p->chip_vendor && + pdev->device == p->chip_device && + pdev->subsystem_vendor == p->subsys_vendor && + pdev->subsystem_device == p->subsys_device && + pdev->revision == p->revision) { + return true; + } + ++p; + } + return false; +} + static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) { + if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) + adev->pm.pp_feature &= ~PP_GFXOFF_MASK; + switch (adev->asic_type) { case CHIP_VEGA10: case CHIP_VEGA12: case CHIP_VEGA20: break; case CHIP_RAVEN: - if (!(adev->rev_id >= 0x8 || - adev->pdev->device == 0x15d8) && - (adev->pm.fw_version < 0x41e2b || /* not raven1 fresh */ - !adev->gfx.rlc.is_rlc_v2_1)) /* without rlc save restore ucodes */ + if (!(adev->rev_id >= 0x8 || adev->pdev->device == 0x15d8) && + ((adev->gfx.rlc_fw_version != 106 && + adev->gfx.rlc_fw_version < 531) || + (adev->gfx.rlc_fw_version == 53815) || + (adev->gfx.rlc_feature_version < 1) || + !adev->gfx.rlc.is_rlc_v2_1)) adev->pm.pp_feature &= ~PP_GFXOFF_MASK; if (adev->pm.pp_feature & PP_GFXOFF_MASK) @@ -1949,6 +1988,17 @@ static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { .query_ras_error_count = &gfx_v9_0_query_ras_error_count }; +static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { + .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, + .select_se_sh = &gfx_v9_0_select_se_sh, + .read_wave_data = &gfx_v9_0_read_wave_data, + .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, + .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, + .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, + .ras_error_inject = &gfx_v9_4_ras_error_inject, + .query_ras_error_count = &gfx_v9_4_query_ras_error_count +}; + static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) { u32 gb_addr_config; @@ -2000,6 +2050,7 @@ static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; break; case CHIP_ARCTURUS: + adev->gfx.funcs = &gfx_v9_4_gfx_funcs; adev->gfx.config.max_hw_contexts = 8; adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; adev->gfx.config.sc_prim_fifo_size_backend = 0x100; @@ -2390,6 +2441,22 @@ static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) } } +static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) +{ + uint32_t tmp; + + switch (adev->asic_type) { + case CHIP_ARCTURUS: + tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); + tmp = REG_SET_FIELD(tmp, SQ_CONFIG, + DISABLE_BARRIER_WAITCNT, 1); + WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); + break; + default: + break; + }; +} + static void gfx_v9_0_constants_init(struct amdgpu_device *adev) { u32 tmp; @@ -2435,6 +2502,7 @@ static void gfx_v9_0_constants_init(struct amdgpu_device *adev) gfx_v9_0_init_compute_vmid(adev); gfx_v9_0_init_gds_vmid(adev); + gfx_v9_0_init_sq_config(adev); } static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) @@ -4029,7 +4097,7 @@ static const struct soc15_reg_entry sgpr2_init_regs[] = { { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, }; -static const struct soc15_reg_entry sec_ded_counter_registers[] = { +static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, @@ -4118,7 +4186,7 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; struct amdgpu_ib ib; struct dma_fence *f = NULL; - int r, i, j, k; + int r, i; unsigned total_size, vgpr_offset, sgpr_offset; u64 gpu_addr; @@ -4264,18 +4332,17 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) goto fail; } - /* read back registers to clear the counters */ - mutex_lock(&adev->grbm_idx_mutex); - for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) { - for (j = 0; j < sec_ded_counter_registers[i].se_num; j++) { - for (k = 0; k < sec_ded_counter_registers[i].instance; k++) { - gfx_v9_0_select_se_sh(adev, j, 0x0, k); - RREG32(SOC15_REG_ENTRY_OFFSET(sec_ded_counter_registers[i])); - } - } + switch (adev->asic_type) + { + case CHIP_VEGA20: + gfx_v9_0_clear_ras_edc_counter(adev); + break; + case CHIP_ARCTURUS: + gfx_v9_4_clear_ras_edc_counter(adev); + break; + default: + break; } - WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); - mutex_unlock(&adev->grbm_idx_mutex); fail: amdgpu_ib_free(adev, &ib, NULL); @@ -4638,7 +4705,7 @@ static int gfx_v9_0_set_powergating_state(void *handle, enum amd_powergating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_PG_STATE_GATE) ? true : false; + bool enable = (state == AMD_PG_STATE_GATE); switch (adev->asic_type) { case CHIP_RAVEN: @@ -4700,7 +4767,7 @@ static int gfx_v9_0_set_clockgating_state(void *handle, case CHIP_ARCTURUS: case CHIP_RENOIR: gfx_v9_0_update_gfx_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; @@ -4717,12 +4784,12 @@ static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) *flags = 0; /* AMD_CG_SUPPORT_GFX_MGCG */ - data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); + data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) *flags |= AMD_CG_SUPPORT_GFX_MGCG; /* AMD_CG_SUPPORT_GFX_CGCG */ - data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); + data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) *flags |= AMD_CG_SUPPORT_GFX_CGCG; @@ -4731,18 +4798,18 @@ static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) *flags |= AMD_CG_SUPPORT_GFX_CGLS; /* AMD_CG_SUPPORT_GFX_RLC_LS */ - data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); + data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; /* AMD_CG_SUPPORT_GFX_CP_LS */ - data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); + data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; if (adev->asic_type != CHIP_ARCTURUS) { /* AMD_CG_SUPPORT_GFX_3D_CGCG */ - data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); + data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; @@ -5213,6 +5280,7 @@ static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigne static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) { struct amdgpu_device *adev = ring->adev; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); amdgpu_ring_write(ring, 0 | /* src: register*/ @@ -5221,9 +5289,9 @@ static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg) amdgpu_ring_write(ring, reg); amdgpu_ring_write(ring, 0); amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + - adev->virt.reg_val_offs * 4)); + kiq->reg_val_offs * 4)); amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + - adev->virt.reg_val_offs * 4)); + kiq->reg_val_offs * 4)); } static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, @@ -5545,7 +5613,7 @@ static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, } -static const struct soc15_ras_field_entry gc_ras_fields_vg20[] = { +static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) @@ -5993,7 +6061,7 @@ static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, int ret; struct ta_ras_trigger_error_input block_info = { 0 }; - if (adev->asic_type != CHIP_VEGA20) + if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) return -EINVAL; if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) @@ -6118,7 +6186,7 @@ static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); - for (i = 0; i < 16; i++) { + for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); @@ -6137,7 +6205,7 @@ static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, } } - for (i = 0; i < 7; i++) { + for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); @@ -6158,7 +6226,7 @@ static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, } } - for (i = 0; i < 4; i++) { + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); @@ -6170,7 +6238,7 @@ static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, } } - for (i = 0; i < 32; i++) { + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); @@ -6197,36 +6265,36 @@ static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, return 0; } -static int __get_ras_error_count(const struct soc15_reg_entry *reg, +static int gfx_v9_0_ras_error_count(const struct soc15_reg_entry *reg, uint32_t se_id, uint32_t inst_id, uint32_t value, uint32_t *sec_count, uint32_t *ded_count) { uint32_t i; uint32_t sec_cnt, ded_cnt; - for (i = 0; i < ARRAY_SIZE(gc_ras_fields_vg20); i++) { - if(gc_ras_fields_vg20[i].reg_offset != reg->reg_offset || - gc_ras_fields_vg20[i].seg != reg->seg || - gc_ras_fields_vg20[i].inst != reg->inst) + for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { + if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || + gfx_v9_0_ras_fields[i].seg != reg->seg || + gfx_v9_0_ras_fields[i].inst != reg->inst) continue; sec_cnt = (value & - gc_ras_fields_vg20[i].sec_count_mask) >> - gc_ras_fields_vg20[i].sec_count_shift; + gfx_v9_0_ras_fields[i].sec_count_mask) >> + gfx_v9_0_ras_fields[i].sec_count_shift; if (sec_cnt) { DRM_INFO("GFX SubBlock %s, Instance[%d][%d], SEC %d\n", - gc_ras_fields_vg20[i].name, + gfx_v9_0_ras_fields[i].name, se_id, inst_id, sec_cnt); *sec_count += sec_cnt; } ded_cnt = (value & - gc_ras_fields_vg20[i].ded_count_mask) >> - gc_ras_fields_vg20[i].ded_count_shift; + gfx_v9_0_ras_fields[i].ded_count_mask) >> + gfx_v9_0_ras_fields[i].ded_count_shift; if (ded_cnt) { DRM_INFO("GFX SubBlock %s, Instance[%d][%d], DED %d\n", - gc_ras_fields_vg20[i].name, + gfx_v9_0_ras_fields[i].name, se_id, inst_id, ded_cnt); *ded_count += ded_cnt; @@ -6236,6 +6304,58 @@ static int __get_ras_error_count(const struct soc15_reg_entry *reg, return 0; } +static void gfx_v9_0_clear_ras_edc_counter(struct amdgpu_device *adev) +{ + int i, j, k; + + /* read back registers to clear the counters */ + mutex_lock(&adev->grbm_idx_mutex); + for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { + for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { + for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { + gfx_v9_0_select_se_sh(adev, j, 0x0, k); + RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); + } + } + } + WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); + mutex_unlock(&adev->grbm_idx_mutex); + + WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); + WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); + + for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { + WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); + RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); + } + + for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { + WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); + RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); + } + + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); + RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); + } + + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); + RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); + } + + WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); +} + static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status) { @@ -6244,7 +6364,7 @@ static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, uint32_t i, j, k; uint32_t reg_value; - if (adev->asic_type != CHIP_VEGA20) + if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) return -EINVAL; err_data->ue_count = 0; @@ -6252,14 +6372,14 @@ static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, mutex_lock(&adev->grbm_idx_mutex); - for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) { - for (j = 0; j < sec_ded_counter_registers[i].se_num; j++) { - for (k = 0; k < sec_ded_counter_registers[i].instance; k++) { + for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { + for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { + for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { gfx_v9_0_select_se_sh(adev, j, 0, k); reg_value = - RREG32(SOC15_REG_ENTRY_OFFSET(sec_ded_counter_registers[i])); + RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); if (reg_value) - __get_ras_error_count(&sec_ded_counter_registers[i], + gfx_v9_0_ras_error_count(&gfx_v9_0_edc_counter_regs[i], j, k, reg_value, &sec_count, &ded_count); } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.c new file mode 100644 index 000000000000..f099f13d7f1e --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.c @@ -0,0 +1,978 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/kernel.h> + +#include "amdgpu.h" +#include "amdgpu_gfx.h" +#include "soc15.h" +#include "soc15d.h" +#include "amdgpu_atomfirmware.h" +#include "amdgpu_pm.h" + +#include "gc/gc_9_4_1_offset.h" +#include "gc/gc_9_4_1_sh_mask.h" +#include "soc15_common.h" + +#include "gfx_v9_4.h" +#include "amdgpu_ras.h" + +static const struct soc15_reg_entry gfx_v9_4_edc_counter_regs[] = { + /* CPC */ + { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1 }, + /* DC */ + { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1 }, + /* CPF */ + { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1 }, + /* GDS */ + { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1 }, + /* SPI */ + { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1 }, + /* SQ */ + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16 }, + /* SQC */ + { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6 }, + { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6 }, + { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6 }, + { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), 0, 4, 6 }, + /* TA */ + { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16 }, + /* TCA */ + { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2 }, + /* TCC */ + { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16 }, + /* TCI */ + { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72 }, + /* TCP */ + { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16 }, + { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16 }, + /* TD */ + { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16 }, + /* GCEA */ + { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32 }, + { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32 }, + { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 1, 32 }, + /* RLC */ + { SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), 0, 1, 1 }, + { SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), 0, 1, 1 }, +}; + +static void gfx_v9_4_select_se_sh(struct amdgpu_device *adev, u32 se_num, + u32 sh_num, u32 instance) +{ + u32 data; + + if (instance == 0xffffffff) + data = REG_SET_FIELD(0, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + else + data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, + instance); + + if (se_num == 0xffffffff) + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, + 1); + else + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); + + if (sh_num == 0xffffffff) + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, + 1); + else + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); + + WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); +} + +static const struct soc15_ras_field_entry gfx_v9_4_ras_fields[] = { + /* CPC */ + { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), + SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), + SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) }, + { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), + SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), + SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) }, + { "CPC_DC_STATE_RAM_ME1", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), + SOC15_REG_FIELD(DC_EDC_STATE_CNT, SEC_COUNT_ME1), + SOC15_REG_FIELD(DC_EDC_STATE_CNT, DED_COUNT_ME1) }, + { "CPC_DC_CSINVOC_RAM_ME1", + SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), + SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, SEC_COUNT_ME1), + SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, DED_COUNT_ME1) }, + { "CPC_DC_RESTORE_RAM_ME1", + SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), + SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, SEC_COUNT_ME1), + SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, DED_COUNT_ME1) }, + { "CPC_DC_CSINVOC_RAM1_ME1", + SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), + SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, SEC_COUNT1_ME1), + SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, DED_COUNT1_ME1) }, + { "CPC_DC_RESTORE_RAM1_ME1", + SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), + SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, SEC_COUNT1_ME1), + SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, DED_COUNT1_ME1) }, + + /* CPF */ + { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), + SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, SEC_COUNT_ME2), + SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, DED_COUNT_ME2) }, + { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), + SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, SEC_COUNT_ME1), + SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, DED_COUNT_ME1) }, + { "CPF_TCIU_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), + SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), + SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) }, + + /* GDS */ + { "GDS_GRBM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), + SOC15_REG_FIELD(GDS_EDC_GRBM_CNT, SEC), + SOC15_REG_FIELD(GDS_EDC_GRBM_CNT, DED) }, + { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), + SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) }, + { "GDS_PHY_CMD_RAM_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), + SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) }, + { "GDS_PHY_DATA_RAM_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), + SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_DED) }, + { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), + SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) }, + { "GDS_ME1_PIPE0_PIPE_MEM", + SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) }, + { "GDS_ME1_PIPE1_PIPE_MEM", + SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) }, + { "GDS_ME1_PIPE2_PIPE_MEM", + SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) }, + { "GDS_ME1_PIPE3_PIPE_MEM", + SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), + SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) }, + + /* SPI */ + { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SEC_COUNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_DED_COUNT) }, + { "SPI_GDS_EXPREQ", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_GDS_EXPREQ_SEC_COUNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_GDS_EXPREQ_DED_COUNT) }, + { "SPI_WB_GRANT_30", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_WB_GRANT_30_SEC_COUNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_WB_GRANT_30_DED_COUNT) }, + { "SPI_WB_GRANT_61", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_WB_GRANT_61_SEC_COUNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_WB_GRANT_61_DED_COUNT) }, + { "SPI_LIFE_CNT", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_LIFE_CNT_SEC_COUNT), + SOC15_REG_FIELD(SPI_EDC_CNT, SPI_LIFE_CNT_DED_COUNT) }, + + /* SQ */ + { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), + SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), + SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) }, + { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), + SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), + SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) }, + { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), + SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), + SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) }, + { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) }, + { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) }, + { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) }, + { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), + SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) }, + + /* SQC */ + { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), + SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) }, + { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) }, + { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) }, + { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) }, + { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) }, + { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) }, + { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) }, + { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), + SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) }, + { "SQC_INST_BANKA_UTCL1_MISS_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + INST_BANKA_UTCL1_MISS_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + INST_BANKA_UTCL1_MISS_FIFO_DED_COUNT) }, + { "SQC_INST_BANKA_MISS_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, INST_BANKA_MISS_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + INST_BANKA_MISS_FIFO_DED_COUNT) }, + { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), + SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) }, + { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), + SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) }, + { "SQC_DATA_BANKA_HIT_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, DATA_BANKA_HIT_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, DATA_BANKA_HIT_FIFO_DED_COUNT) }, + { "SQC_DATA_BANKA_MISS_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, DATA_BANKA_MISS_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + DATA_BANKA_MISS_FIFO_DED_COUNT) }, + { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), + SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) }, + { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), + SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) }, + { "SQC_INST_BANKB_UTCL1_MISS_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + INST_BANKB_UTCL1_MISS_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + INST_BANKB_UTCL1_MISS_FIFO_DED_COUNT) }, + { "SQC_INST_BANKB_MISS_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, INST_BANKB_MISS_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + INST_BANKB_MISS_FIFO_DED_COUNT) }, + { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), + SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) }, + { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), + SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) }, + { "SQC_DATA_BANKB_HIT_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, DATA_BANKB_HIT_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, DATA_BANKB_HIT_FIFO_DED_COUNT) }, + { "SQC_DATA_BANKB_MISS_FIFO", + SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_PARITY_CNT3), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, DATA_BANKB_MISS_FIFO_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_PARITY_CNT3, + DATA_BANKB_MISS_FIFO_DED_COUNT) }, + { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), + SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), + SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) }, + + /* TA */ + { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) }, + { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SEC_COUNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_DED_COUNT) }, + { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SEC_COUNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_DED_COUNT) }, + { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SEC_COUNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_DED_COUNT) }, + { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SEC_COUNT), + SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_DED_COUNT) }, + + /* TCA */ + { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), + SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_DED_COUNT) }, + { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), + SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_DED_COUNT) }, + + /* TCC */ + { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), + SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) }, + { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), + SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) }, + { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), + SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) }, + { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), + SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) }, + { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, IN_USE_DEC_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, IN_USE_DEC_DED_COUNT) }, + { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, IN_USE_TRANSFER_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, IN_USE_TRANSFER_DED_COUNT) }, + { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, RETURN_DATA_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, RETURN_DATA_DED_COUNT) }, + { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, RETURN_CONTROL_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, RETURN_CONTROL_DED_COUNT) }, + { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, UC_ATOMIC_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, UC_ATOMIC_FIFO_DED_COUNT) }, + { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_DED_COUNT) }, + { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_DED_COUNT) }, + { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), + SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) }, + { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), + SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_DED_COUNT) }, + { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), + SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_DED_COUNT) }, + { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), + SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_NEXT_RAM_SEC_COUNT), + SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_NEXT_RAM_DED_COUNT) }, + + /* TCI */ + { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), + SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SEC_COUNT), + SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_DED_COUNT) }, + + /* TCP */ + { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) }, + { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) }, + { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_DED_COUNT) }, + { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_DED_COUNT) }, + { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 0, 0 }, + { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) }, + { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), + SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) }, + + /* TD */ + { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), + SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), + SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) }, + { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), + SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), + SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) }, + { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), + SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SEC_COUNT), + SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_DED_COUNT) }, + + /* EA */ + { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) }, + { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) }, + { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) }, + { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) }, + { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) }, + { "EA_GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) }, + { "EA_GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) }, + { "EA_GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) }, + { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 0, 0 }, + { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, DRAMRD_PAGEMEM_DED_COUNT) }, + { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 0, 0 }, + { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, DRAMWR_PAGEMEM_DED_COUNT) }, + { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 0, 0 }, + { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, IORD_CMDMEM_DED_COUNT) }, + { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 0, 0 }, + { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, IOWR_CMDMEM_DED_COUNT) }, + { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 0, 0 }, + { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, IOWR_DATAMEM_DED_COUNT) }, + { "EA_GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 0, 0 }, + { "EA_GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, GMIRD_PAGEMEM_DED_COUNT) }, + { "EA_GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 0, 0 }, + { "EA_GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT) }, + { "EA_MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_DED_COUNT) }, + { "EA_MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_DED_COUNT) }, + { "EA_MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_DED_COUNT) }, + { "EA_MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_DED_COUNT) }, + { "EA_MAM_A0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A0MEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A0MEM_DED_COUNT) }, + { "EA_MAM_A1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A1MEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A1MEM_DED_COUNT) }, + { "EA_MAM_A2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A2MEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A2MEM_DED_COUNT) }, + { "EA_MAM_A3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A3MEM_SEC_COUNT), + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_A3MEM_DED_COUNT) }, + { "EA_MAM_AFMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), + SOC15_REG_FIELD(GCEA_EDC_CNT, MAM_AFMEM_SEC_COUNT), 0, 0 }, + { "EA_MAM_AFMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT3), 0, 0, + SOC15_REG_FIELD(GCEA_EDC_CNT3, MAM_AFMEM_DED_COUNT) }, + + /* RLC */ + { "RLCG_INSTR_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCG_INSTR_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCG_INSTR_RAM_DED_COUNT) }, + { "RLCG_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCG_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCG_SCRATCH_RAM_DED_COUNT) }, + { "RLCV_INSTR_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCV_INSTR_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCV_INSTR_RAM_DED_COUNT) }, + { "RLCV_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCV_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLCV_SCRATCH_RAM_DED_COUNT) }, + { "RLC_TCTAG_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_TCTAG_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_TCTAG_RAM_DED_COUNT) }, + { "RLC_SPM_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_SPM_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_SPM_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SRM_DATA_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_SRM_DATA_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_SRM_DATA_RAM_DED_COUNT) }, + { "RLC_SRM_ADDR_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_SRM_ADDR_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT, RLC_SRM_ADDR_RAM_DED_COUNT) }, + { "RLC_SPM_SE0_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE0_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE0_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SPM_SE1_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE1_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE1_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SPM_SE2_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE2_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE2_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SPM_SE3_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE3_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE3_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SPM_SE4_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE4_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE4_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SPM_SE5_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE5_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE5_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SPM_SE6_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE6_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE6_SCRATCH_RAM_DED_COUNT) }, + { "RLC_SPM_SE7_SCRATCH_RAM", SOC15_REG_ENTRY(GC, 0, mmRLC_EDC_CNT2), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE7_SCRATCH_RAM_SEC_COUNT), + SOC15_REG_FIELD(RLC_EDC_CNT2, RLC_SPM_SE7_SCRATCH_RAM_DED_COUNT) }, +}; + +static const char * const vml2_mems[] = { + "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", + "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", + "UTC_VML2_BANK_CACHE_0_4K_MEM0", + "UTC_VML2_BANK_CACHE_0_4K_MEM1", + "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", + "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", + "UTC_VML2_BANK_CACHE_1_4K_MEM0", + "UTC_VML2_BANK_CACHE_1_4K_MEM1", + "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", + "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", + "UTC_VML2_BANK_CACHE_2_4K_MEM0", + "UTC_VML2_BANK_CACHE_2_4K_MEM1", + "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", + "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", + "UTC_VML2_BANK_CACHE_3_4K_MEM0", + "UTC_VML2_BANK_CACHE_3_4K_MEM1", + "UTC_VML2_IFIFO_GROUP0", + "UTC_VML2_IFIFO_GROUP1", + "UTC_VML2_IFIFO_GROUP2", + "UTC_VML2_IFIFO_GROUP3", + "UTC_VML2_IFIFO_GROUP4", + "UTC_VML2_IFIFO_GROUP5", + "UTC_VML2_IFIFO_GROUP6", + "UTC_VML2_IFIFO_GROUP7", + "UTC_VML2_IFIFO_GROUP8", + "UTC_VML2_IFIFO_GROUP9", + "UTC_VML2_IFIFO_GROUP10", + "UTC_VML2_IFIFO_GROUP11", + "UTC_VML2_IFIFO_GROUP12", + "UTC_VML2_IFIFO_GROUP13", + "UTC_VML2_IFIFO_GROUP14", + "UTC_VML2_IFIFO_GROUP15", + "UTC_VML2_IFIFO_GROUP16", + "UTC_VML2_IFIFO_GROUP17", + "UTC_VML2_IFIFO_GROUP18", + "UTC_VML2_IFIFO_GROUP19", + "UTC_VML2_IFIFO_GROUP20", + "UTC_VML2_IFIFO_GROUP21", + "UTC_VML2_IFIFO_GROUP22", + "UTC_VML2_IFIFO_GROUP23", + "UTC_VML2_IFIFO_GROUP24", +}; + +static const char * const vml2_walker_mems[] = { + "UTC_VML2_CACHE_PDE0_MEM0", + "UTC_VML2_CACHE_PDE0_MEM1", + "UTC_VML2_CACHE_PDE1_MEM0", + "UTC_VML2_CACHE_PDE1_MEM1", + "UTC_VML2_CACHE_PDE2_MEM0", + "UTC_VML2_CACHE_PDE2_MEM1", + "UTC_VML2_RDIF_ARADDRS", + "UTC_VML2_RDIF_LOG_FIFO", + "UTC_VML2_QUEUE_REQ", + "UTC_VML2_QUEUE_RET", +}; + +static const char * const utcl2_router_mems[] = { + "UTCL2_ROUTER_GROUP0_VML2_REQ_FIFO0", + "UTCL2_ROUTER_GROUP1_VML2_REQ_FIFO1", + "UTCL2_ROUTER_GROUP2_VML2_REQ_FIFO2", + "UTCL2_ROUTER_GROUP3_VML2_REQ_FIFO3", + "UTCL2_ROUTER_GROUP4_VML2_REQ_FIFO4", + "UTCL2_ROUTER_GROUP5_VML2_REQ_FIFO5", + "UTCL2_ROUTER_GROUP6_VML2_REQ_FIFO6", + "UTCL2_ROUTER_GROUP7_VML2_REQ_FIFO7", + "UTCL2_ROUTER_GROUP8_VML2_REQ_FIFO8", + "UTCL2_ROUTER_GROUP9_VML2_REQ_FIFO9", + "UTCL2_ROUTER_GROUP10_VML2_REQ_FIFO10", + "UTCL2_ROUTER_GROUP11_VML2_REQ_FIFO11", + "UTCL2_ROUTER_GROUP12_VML2_REQ_FIFO12", + "UTCL2_ROUTER_GROUP13_VML2_REQ_FIFO13", + "UTCL2_ROUTER_GROUP14_VML2_REQ_FIFO14", + "UTCL2_ROUTER_GROUP15_VML2_REQ_FIFO15", + "UTCL2_ROUTER_GROUP16_VML2_REQ_FIFO16", + "UTCL2_ROUTER_GROUP17_VML2_REQ_FIFO17", + "UTCL2_ROUTER_GROUP18_VML2_REQ_FIFO18", + "UTCL2_ROUTER_GROUP19_VML2_REQ_FIFO19", + "UTCL2_ROUTER_GROUP20_VML2_REQ_FIFO20", + "UTCL2_ROUTER_GROUP21_VML2_REQ_FIFO21", + "UTCL2_ROUTER_GROUP22_VML2_REQ_FIFO22", + "UTCL2_ROUTER_GROUP23_VML2_REQ_FIFO23", + "UTCL2_ROUTER_GROUP24_VML2_REQ_FIFO24", +}; + +static const char * const atc_l2_cache_2m_mems[] = { + "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", + "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", + "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", + "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", +}; + +static const char * const atc_l2_cache_4k_mems[] = { + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", + "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", + "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", + "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", + "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", +}; + +static int gfx_v9_4_query_utc_edc_status(struct amdgpu_device *adev, + struct ras_err_data *err_data) +{ + uint32_t i, data; + uint32_t sec_count, ded_count; + + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_CNTL, 0); + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_CNTL, 0); + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_CNTL, 0); + + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_CNTL, 0); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_CNTL, 0); + + for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_INDEX, i); + data = RREG32_SOC15(GC, 0, mmVML2_MEM_ECC_CNTL); + + sec_count = REG_GET_FIELD(data, VML2_MEM_ECC_CNTL, SEC_COUNT); + if (sec_count) { + DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, + vml2_mems[i], sec_count); + err_data->ce_count += sec_count; + } + + ded_count = REG_GET_FIELD(data, VML2_MEM_ECC_CNTL, DED_COUNT); + if (ded_count) { + DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, + vml2_mems[i], ded_count); + err_data->ue_count += ded_count; + } + } + + for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_INDEX, i); + data = RREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_CNTL); + + sec_count = REG_GET_FIELD(data, VML2_WALKER_MEM_ECC_CNTL, + SEC_COUNT); + if (sec_count) { + DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, + vml2_walker_mems[i], sec_count); + err_data->ce_count += sec_count; + } + + ded_count = REG_GET_FIELD(data, VML2_WALKER_MEM_ECC_CNTL, + DED_COUNT); + if (ded_count) { + DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, + vml2_walker_mems[i], ded_count); + err_data->ue_count += ded_count; + } + } + + for (i = 0; i < ARRAY_SIZE(utcl2_router_mems); i++) { + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_INDEX, i); + data = RREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_CNTL); + + sec_count = REG_GET_FIELD(data, UTCL2_MEM_ECC_CNTL, SEC_COUNT); + if (sec_count) { + DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, + utcl2_router_mems[i], sec_count); + err_data->ce_count += sec_count; + } + + ded_count = REG_GET_FIELD(data, UTCL2_MEM_ECC_CNTL, DED_COUNT); + if (ded_count) { + DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, + utcl2_router_mems[i], ded_count); + err_data->ue_count += ded_count; + } + } + + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_INDEX, i); + data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_CNTL); + + sec_count = REG_GET_FIELD(data, ATC_L2_CACHE_2M_DSM_CNTL, + SEC_COUNT); + if (sec_count) { + DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, + atc_l2_cache_2m_mems[i], sec_count); + err_data->ce_count += sec_count; + } + + ded_count = REG_GET_FIELD(data, ATC_L2_CACHE_2M_DSM_CNTL, + DED_COUNT); + if (ded_count) { + DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, + atc_l2_cache_2m_mems[i], ded_count); + err_data->ue_count += ded_count; + } + } + + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_INDEX, i); + data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_CNTL); + + sec_count = REG_GET_FIELD(data, ATC_L2_CACHE_4K_DSM_CNTL, + SEC_COUNT); + if (sec_count) { + DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i, + atc_l2_cache_4k_mems[i], sec_count); + err_data->ce_count += sec_count; + } + + ded_count = REG_GET_FIELD(data, ATC_L2_CACHE_4K_DSM_CNTL, + DED_COUNT); + if (ded_count) { + DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i, + atc_l2_cache_4k_mems[i], ded_count); + err_data->ue_count += ded_count; + } + } + + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_INDEX, 255); + + return 0; +} + +static int gfx_v9_4_ras_error_count(const struct soc15_reg_entry *reg, + uint32_t se_id, uint32_t inst_id, + uint32_t value, uint32_t *sec_count, + uint32_t *ded_count) +{ + uint32_t i; + uint32_t sec_cnt, ded_cnt; + + for (i = 0; i < ARRAY_SIZE(gfx_v9_4_ras_fields); i++) { + if (gfx_v9_4_ras_fields[i].reg_offset != reg->reg_offset || + gfx_v9_4_ras_fields[i].seg != reg->seg || + gfx_v9_4_ras_fields[i].inst != reg->inst) + continue; + + sec_cnt = (value & gfx_v9_4_ras_fields[i].sec_count_mask) >> + gfx_v9_4_ras_fields[i].sec_count_shift; + if (sec_cnt) { + DRM_INFO("GFX SubBlock %s, Instance[%d][%d], SEC %d\n", + gfx_v9_4_ras_fields[i].name, se_id, inst_id, + sec_cnt); + *sec_count += sec_cnt; + } + + ded_cnt = (value & gfx_v9_4_ras_fields[i].ded_count_mask) >> + gfx_v9_4_ras_fields[i].ded_count_shift; + if (ded_cnt) { + DRM_INFO("GFX SubBlock %s, Instance[%d][%d], DED %d\n", + gfx_v9_4_ras_fields[i].name, se_id, inst_id, + ded_cnt); + *ded_count += ded_cnt; + } + } + + return 0; +} + +int gfx_v9_4_query_ras_error_count(struct amdgpu_device *adev, + void *ras_error_status) +{ + struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; + uint32_t sec_count = 0, ded_count = 0; + uint32_t i, j, k; + uint32_t reg_value; + + if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) + return -EINVAL; + + err_data->ue_count = 0; + err_data->ce_count = 0; + + mutex_lock(&adev->grbm_idx_mutex); + + for (i = 0; i < ARRAY_SIZE(gfx_v9_4_edc_counter_regs); i++) { + for (j = 0; j < gfx_v9_4_edc_counter_regs[i].se_num; j++) { + for (k = 0; k < gfx_v9_4_edc_counter_regs[i].instance; + k++) { + gfx_v9_4_select_se_sh(adev, j, 0, k); + reg_value = RREG32(SOC15_REG_ENTRY_OFFSET( + gfx_v9_4_edc_counter_regs[i])); + if (reg_value) + gfx_v9_4_ras_error_count( + &gfx_v9_4_edc_counter_regs[i], + j, k, reg_value, &sec_count, + &ded_count); + } + } + } + + err_data->ce_count += sec_count; + err_data->ue_count += ded_count; + + gfx_v9_4_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + mutex_unlock(&adev->grbm_idx_mutex); + + gfx_v9_4_query_utc_edc_status(adev, err_data); + + return 0; +} + +void gfx_v9_4_clear_ras_edc_counter(struct amdgpu_device *adev) +{ + int i, j, k; + + mutex_lock(&adev->grbm_idx_mutex); + for (i = 0; i < ARRAY_SIZE(gfx_v9_4_edc_counter_regs); i++) { + for (j = 0; j < gfx_v9_4_edc_counter_regs[i].se_num; j++) { + for (k = 0; k < gfx_v9_4_edc_counter_regs[i].instance; + k++) { + gfx_v9_4_select_se_sh(adev, j, 0x0, k); + RREG32(SOC15_REG_ENTRY_OFFSET( + gfx_v9_4_edc_counter_regs[i])); + } + } + } + WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); + mutex_unlock(&adev->grbm_idx_mutex); + + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_CNTL, 0); + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_CNTL, 0); + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_CNTL, 0); + + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_CNTL, 0); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_CNTL, 0); + + for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_INDEX, i); + RREG32_SOC15(GC, 0, mmVML2_MEM_ECC_CNTL); + } + + for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_INDEX, i); + RREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_CNTL); + } + + for (i = 0; i < ARRAY_SIZE(utcl2_router_mems); i++) { + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_INDEX, i); + RREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_CNTL); + } + + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_INDEX, i); + RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_CNTL); + } + + for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_INDEX, i); + RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_CNTL); + } + + WREG32_SOC15(GC, 0, mmVML2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmVML2_WALKER_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmUTCL2_MEM_ECC_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_DSM_INDEX, 255); + WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_DSM_INDEX, 255); +} + +int gfx_v9_4_ras_error_inject(struct amdgpu_device *adev, void *inject_if) +{ + struct ras_inject_if *info = (struct ras_inject_if *)inject_if; + int ret; + struct ta_ras_trigger_error_input block_info = { 0 }; + + if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) + return -EINVAL; + + block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); + block_info.sub_block_index = info->head.sub_block_index; + block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); + block_info.address = info->address; + block_info.value = info->value; + + mutex_lock(&adev->grbm_idx_mutex); + ret = psp_ras_trigger_error(&adev->psp, &block_info); + mutex_unlock(&adev->grbm_idx_mutex); + + return ret; +} diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.h new file mode 100644 index 000000000000..2e3f6f755ad4 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4.h @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __GFX_V9_4_H__ +#define __GFX_V9_4_H__ + +void gfx_v9_4_clear_ras_edc_counter(struct amdgpu_device *adev); + +int gfx_v9_4_query_ras_error_count(struct amdgpu_device *adev, + void *ras_error_status); + +int gfx_v9_4_ras_error_inject(struct amdgpu_device *adev, + void *inject_if); + +#endif /* __GFX_V9_4_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index bbede09983e1..9775eca6fe43 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -262,7 +262,8 @@ static void gmc_v10_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, { bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(adev, vmhub); struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; - u32 tmp = gmc_v10_0_get_invalidate_req(vmid, flush_type); + u32 inv_req = gmc_v10_0_get_invalidate_req(vmid, flush_type); + u32 tmp; /* Use register 17 for GART */ const unsigned eng = 17; unsigned int i; @@ -289,7 +290,7 @@ static void gmc_v10_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); } - WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); + WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, inv_req); /* * Issue a dummy read to wait for the ACK register to be cleared @@ -418,7 +419,8 @@ static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, if (amdgpu_emu_mode == 0 && ring->sched.ready) { spin_lock(&adev->gfx.kiq.ring_lock); - amdgpu_ring_alloc(ring, kiq->pmf->invalidate_tlbs_size); + /* 2 dwords flush + 8 dwords fence */ + amdgpu_ring_alloc(ring, kiq->pmf->invalidate_tlbs_size + 8); kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); amdgpu_fence_emit_polling(ring, &seq); @@ -441,10 +443,10 @@ static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, if (all_hub) { for (i = 0; i < adev->num_vmhubs; i++) gmc_v10_0_flush_gpu_tlb(adev, vmid, - i, 0); + i, flush_type); } else { gmc_v10_0_flush_gpu_tlb(adev, vmid, - AMDGPU_GFXHUB_0, 0); + AMDGPU_GFXHUB_0, flush_type); } break; } @@ -640,12 +642,7 @@ static int gmc_v10_0_late_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; int r; - /* - * Can't free the stolen VGA memory when it might be used for memory - * training again. - */ - if (!adev->fw_vram_usage.mem_train_support) - amdgpu_bo_late_init(adev); + amdgpu_bo_late_init(adev); r = amdgpu_gmc_allocate_vm_inv_eng(adev); if (r) @@ -829,19 +826,6 @@ static int gmc_v10_0_sw_init(void *handle) adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev); - /* - * In dual GPUs scenario, stolen_size is assigned to zero on the - * secondary GPU, since there is no pre-OS console using that memory. - * Then the bottom region of VRAM was allocated as GTT, unfortunately a - * small region of bottom VRAM was encroached by UMC firmware during - * GDDR6 BIST training, this cause page fault. - * The page fault can be fixed by forcing stolen_size to 3MB, then the - * bottom region of VRAM was allocated as stolen memory, GTT corruption - * avoid. - */ - adev->gmc.stolen_size = max(adev->gmc.stolen_size, - AMDGPU_STOLEN_BIST_TRAINING_DEFAULT_SIZE); - /* Memory manager */ r = amdgpu_bo_init(adev); if (r) @@ -881,13 +865,6 @@ static void gmc_v10_0_gart_fini(struct amdgpu_device *adev) static int gmc_v10_0_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - void *stolen_vga_buf; - - /* - * Free the stolen memory if it wasn't already freed in late_init - * because of memory training. - */ - amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf); amdgpu_vm_manager_fini(adev); gmc_v10_0_gart_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 19d5b133e1d7..9da9596a3638 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -381,7 +381,8 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev) adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); #ifdef CONFIG_X86_64 - if (adev->flags & AMD_IS_APU) { + if (adev->flags & AMD_IS_APU && + adev->gmc.real_vram_size > adev->gmc.aper_size) { adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22; adev->gmc.aper_size = adev->gmc.real_vram_size; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 40a496804356..90216abf14a4 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -476,13 +476,13 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, { bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); const unsigned eng = 17; - u32 j, tmp; + u32 j, inv_req, tmp; struct amdgpu_vmhub *hub; BUG_ON(vmhub >= adev->num_vmhubs); hub = &adev->vmhub[vmhub]; - tmp = gmc_v9_0_get_invalidate_req(vmid, flush_type); + inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); /* This is necessary for a HW workaround under SRIOV as well * as GFXOFF under bare metal @@ -493,7 +493,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, uint32_t req = hub->vm_inv_eng0_req + eng; uint32_t ack = hub->vm_inv_eng0_ack + eng; - amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, tmp, + amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 1 << vmid); return; } @@ -521,7 +521,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); } - WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); + WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, inv_req); /* * Issue a dummy read to wait for the ACK register to be cleared @@ -578,7 +578,8 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, if (ring->sched.ready) { spin_lock(&adev->gfx.kiq.ring_lock); - amdgpu_ring_alloc(ring, kiq->pmf->invalidate_tlbs_size); + /* 2 dwords flush + 8 dwords fence */ + amdgpu_ring_alloc(ring, kiq->pmf->invalidate_tlbs_size + 8); kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); amdgpu_fence_emit_polling(ring, &seq); @@ -601,10 +602,10 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, if (all_hub) { for (i = 0; i < adev->num_vmhubs; i++) gmc_v9_0_flush_gpu_tlb(adev, vmid, - i, 0); + i, flush_type); } else { gmc_v9_0_flush_gpu_tlb(adev, vmid, - AMDGPU_GFXHUB_0, 0); + AMDGPU_GFXHUB_0, flush_type); } break; } diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c index a78292d84854..ff2e6e1ccde7 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c @@ -690,7 +690,7 @@ static int jpeg_v2_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); if (enable) { if (jpeg_v2_0_is_idle(handle)) diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c index 2c58939e6ad0..c6d046df4b70 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c @@ -469,7 +469,7 @@ static int jpeg_v2_5_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); int i; for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c index adfd8a6171eb..49a3a56ec017 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c @@ -523,9 +523,9 @@ int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev, case CHIP_RAVEN: case CHIP_RENOIR: mmhub_v1_0_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); mmhub_v1_0_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c index a7cb185d639a..bde189680521 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c @@ -427,9 +427,9 @@ int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev, case CHIP_NAVI14: case CHIP_NAVI12: mmhub_v2_0_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); mmhub_v2_0_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c index 5c42387c9274..a5281df8d84f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c @@ -625,9 +625,9 @@ int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev, switch (adev->asic_type) { case CHIP_ARCTURUS: mmhub_v9_4_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); mmhub_v9_4_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; @@ -663,6 +663,7 @@ void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u32 *flags) } static const struct soc15_ras_field_entry mmhub_v9_4_ras_fields[] = { + /* MMHUB Range 0 */ { "MMEA0_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT), SOC15_REG_FIELD(MMEA0_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), SOC15_REG_FIELD(MMEA0_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), @@ -751,6 +752,24 @@ static const struct soc15_ras_field_entry mmhub_v9_4_ras_fields[] = { 0, 0, SOC15_REG_FIELD(MMEA0_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), }, + { "MMEA0_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT2), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA0_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT2), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA0_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT2), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA0_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT2), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA0_EDC_CNT2, MAM_D3MEM_DED_COUNT), + }, + + /* MMHUB Range 1 */ { "MMEA1_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT), SOC15_REG_FIELD(MMEA1_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), SOC15_REG_FIELD(MMEA1_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), @@ -838,16 +857,686 @@ static const struct soc15_ras_field_entry mmhub_v9_4_ras_fields[] = { { "MMEA1_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT3), 0, 0, SOC15_REG_FIELD(MMEA1_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), + }, + { "MMEA1_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT2), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA1_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT2), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA1_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT2), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA1_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT2), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA1_EDC_CNT2, MAM_D3MEM_DED_COUNT), + }, + + /* MMHAB Range 2*/ + { "MMEA2_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), + }, + { "MMEA2_DRAMWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT), + }, + { "MMEA2_DRAMWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT), + }, + { "MMEA2_RRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, RRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, RRET_TAGMEM_DED_COUNT), + }, + { "MMEA2_WRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, WRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, WRET_TAGMEM_DED_COUNT), + }, + { "MMEA2_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA2_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA2_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, IORD_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA2_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, IOWR_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA2_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT, IOWR_DATAMEM_SED_COUNT), + 0, 0, + }, + { "MMEA2_GMIRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT), + }, + { "MMEA2_GMIWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT), + }, + { "MMEA2_GMIWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT), + }, + { "MMEA2_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA2_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA2_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA2_EDC_CNT3, DRAMRD_PAGEMEM_DED_COUNT), + }, + { "MMEA2_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA2_EDC_CNT3, DRAMWR_PAGEMEM_DED_COUNT), + }, + { "MMEA2_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA2_EDC_CNT3, IORD_CMDMEM_DED_COUNT), + }, + { "MMEA2_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA2_EDC_CNT3, IOWR_CMDMEM_DED_COUNT), + }, + { "MMEA2_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA2_EDC_CNT3, IOWR_DATAMEM_DED_COUNT), + }, + { "MMEA2_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA2_EDC_CNT3, GMIRD_PAGEMEM_DED_COUNT), + }, + { "MMEA2_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA2_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), + }, + { "MMEA2_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA2_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA2_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA2_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA2_EDC_CNT2, MAM_D3MEM_DED_COUNT), + }, + + /* MMHUB Rang 3 */ + { "MMEA3_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), + }, + { "MMEA3_DRAMWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT), + }, + { "MMEA3_DRAMWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT), + }, + { "MMEA3_RRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, RRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, RRET_TAGMEM_DED_COUNT), + }, + { "MMEA3_WRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, WRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, WRET_TAGMEM_DED_COUNT), + }, + { "MMEA3_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA3_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA3_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, IORD_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA3_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, IOWR_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA3_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT, IOWR_DATAMEM_SED_COUNT), + 0, 0, + }, + { "MMEA3_GMIRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT), + }, + { "MMEA3_GMIWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT), + }, + { "MMEA3_GMIWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT), + }, + { "MMEA3_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA3_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA3_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA3_EDC_CNT3, DRAMRD_PAGEMEM_DED_COUNT), + }, + { "MMEA3_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA3_EDC_CNT3, DRAMWR_PAGEMEM_DED_COUNT), + }, + { "MMEA3_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA3_EDC_CNT3, IORD_CMDMEM_DED_COUNT), + }, + { "MMEA3_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA3_EDC_CNT3, IOWR_CMDMEM_DED_COUNT), + }, + { "MMEA3_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA3_EDC_CNT3, IOWR_DATAMEM_DED_COUNT), + }, + { "MMEA3_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA3_EDC_CNT3, GMIRD_PAGEMEM_DED_COUNT), + }, + { "MMEA3_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA3_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), + }, + { "MMEA3_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA3_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA3_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA3_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA3_EDC_CNT2, MAM_D3MEM_DED_COUNT), + }, + + /* MMHUB Range 4 */ + { "MMEA4_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), + }, + { "MMEA4_DRAMWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT), + }, + { "MMEA4_DRAMWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT), + }, + { "MMEA4_RRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, RRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, RRET_TAGMEM_DED_COUNT), + }, + { "MMEA4_WRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, WRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, WRET_TAGMEM_DED_COUNT), + }, + { "MMEA4_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA4_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA4_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, IORD_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA4_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, IOWR_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA4_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT, IOWR_DATAMEM_SED_COUNT), + 0, 0, + }, + { "MMEA4_GMIRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT), + }, + { "MMEA4_GMIWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT), + }, + { "MMEA4_GMIWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT), + }, + { "MMEA4_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA4_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA4_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA4_EDC_CNT3, DRAMRD_PAGEMEM_DED_COUNT), + }, + { "MMEA4_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA4_EDC_CNT3, DRAMWR_PAGEMEM_DED_COUNT), + }, + { "MMEA4_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA4_EDC_CNT3, IORD_CMDMEM_DED_COUNT), + }, + { "MMEA4_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA4_EDC_CNT3, IOWR_CMDMEM_DED_COUNT), + }, + { "MMEA4_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA4_EDC_CNT3, IOWR_DATAMEM_DED_COUNT), + }, + { "MMEA4_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA4_EDC_CNT3, GMIRD_PAGEMEM_DED_COUNT), + }, + { "MMEA4_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA4_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), + }, + { "MMEA4_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA4_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA4_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA4_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA4_EDC_CNT2, MAM_D3MEM_DED_COUNT), + }, + + /* MMHUAB Range 5 */ + { "MMEA5_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), + }, + { "MMEA5_DRAMWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT), + }, + { "MMEA5_DRAMWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT), + }, + { "MMEA5_RRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, RRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, RRET_TAGMEM_DED_COUNT), + }, + { "MMEA5_WRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, WRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, WRET_TAGMEM_DED_COUNT), + }, + { "MMEA5_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA5_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA5_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, IORD_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA5_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, IOWR_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA5_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT, IOWR_DATAMEM_SED_COUNT), + 0, 0, + }, + { "MMEA5_GMIRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT), + }, + { "MMEA5_GMIWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT), + }, + { "MMEA5_GMIWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT), + }, + { "MMEA5_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA5_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA5_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA5_EDC_CNT3, DRAMRD_PAGEMEM_DED_COUNT), + }, + { "MMEA5_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA5_EDC_CNT3, DRAMWR_PAGEMEM_DED_COUNT), + }, + { "MMEA5_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA5_EDC_CNT3, IORD_CMDMEM_DED_COUNT), + }, + { "MMEA5_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA5_EDC_CNT3, IOWR_CMDMEM_DED_COUNT), + }, + { "MMEA5_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA5_EDC_CNT3, IOWR_DATAMEM_DED_COUNT), + }, + { "MMEA5_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA5_EDC_CNT3, GMIRD_PAGEMEM_DED_COUNT), + }, + { "MMEA5_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA5_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), + }, + { "MMEA5_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA5_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA5_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA5_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA5_EDC_CNT2, MAM_D3MEM_DED_COUNT), + }, + + /* MMHUB Range 6 */ + { "MMEA6_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), + }, + { "MMEA6_DRAMWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT), + }, + { "MMEA6_DRAMWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT), + }, + { "MMEA6_RRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, RRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, RRET_TAGMEM_DED_COUNT), + }, + { "MMEA6_WRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, WRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, WRET_TAGMEM_DED_COUNT), + }, + { "MMEA6_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA6_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA6_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, IORD_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA6_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, IOWR_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA6_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT, IOWR_DATAMEM_SED_COUNT), + 0, 0, + }, + { "MMEA6_GMIRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT), + }, + { "MMEA6_GMIWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT), + }, + { "MMEA6_GMIWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT), + }, + { "MMEA6_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA6_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA6_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA6_EDC_CNT3, DRAMRD_PAGEMEM_DED_COUNT), + }, + { "MMEA6_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA6_EDC_CNT3, DRAMWR_PAGEMEM_DED_COUNT), + }, + { "MMEA6_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA6_EDC_CNT3, IORD_CMDMEM_DED_COUNT), + }, + { "MMEA6_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA6_EDC_CNT3, IOWR_CMDMEM_DED_COUNT), + }, + { "MMEA6_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA6_EDC_CNT3, IOWR_DATAMEM_DED_COUNT), + }, + { "MMEA6_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA6_EDC_CNT3, GMIRD_PAGEMEM_DED_COUNT), + }, + { "MMEA6_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA6_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), + }, + { "MMEA6_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA6_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA6_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA6_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA6_EDC_CNT2, MAM_D3MEM_DED_COUNT), + }, + + /* MMHUB Range 7*/ + { "MMEA7_DRAMRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT), + }, + { "MMEA7_DRAMWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT), + }, + { "MMEA7_DRAMWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT), + }, + { "MMEA7_RRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, RRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, RRET_TAGMEM_DED_COUNT), + }, + { "MMEA7_WRET_TAGMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, WRET_TAGMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, WRET_TAGMEM_DED_COUNT), + }, + { "MMEA7_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA7_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA7_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, IORD_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA7_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, IOWR_CMDMEM_SED_COUNT), + 0, 0, + }, + { "MMEA7_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT, IOWR_DATAMEM_SED_COUNT), + 0, 0, + }, + { "MMEA7_GMIRD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT), + }, + { "MMEA7_GMIWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT), + }, + { "MMEA7_GMIWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT), + }, + { "MMEA7_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA7_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), + 0, 0, + }, + { "MMEA7_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA7_EDC_CNT3, DRAMRD_PAGEMEM_DED_COUNT), + }, + { "MMEA7_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA7_EDC_CNT3, DRAMWR_PAGEMEM_DED_COUNT), + }, + { "MMEA7_IORD_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA7_EDC_CNT3, IORD_CMDMEM_DED_COUNT), + }, + { "MMEA7_IOWR_CMDMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA7_EDC_CNT3, IOWR_CMDMEM_DED_COUNT), + }, + { "MMEA7_IOWR_DATAMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA7_EDC_CNT3, IOWR_DATAMEM_DED_COUNT), + }, + { "MMEA7_GMIRD_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA7_EDC_CNT3, GMIRD_PAGEMEM_DED_COUNT), + }, + { "MMEA7_GMIWR_PAGEMEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), + 0, 0, + SOC15_REG_FIELD(MMEA7_EDC_CNT3, GMIWR_PAGEMEM_DED_COUNT), + }, + { "MMEA7_MAM_D0MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D0MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D0MEM_DED_COUNT), + }, + { "MMEA7_MAM_D1MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D1MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D1MEM_DED_COUNT), + }, + { "MMEA7_MAM_D2MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D2MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D2MEM_DED_COUNT), + }, + { "MMEA7_MAM_D3MEM", SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D3MEM_SED_COUNT), + SOC15_REG_FIELD(MMEA7_EDC_CNT2, MAM_D3MEM_DED_COUNT), } }; static const struct soc15_reg_entry mmhub_v9_4_edc_cnt_regs[] = { - { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT), 0, 0, 0}, - { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT2), 0, 0, 0}, - { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT3), 0, 0, 0}, - { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT), 0, 0, 0}, - { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT2), 0, 0, 0}, - { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT3), 0, 0, 0}, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA0_EDC_CNT3), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA1_EDC_CNT3), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA2_EDC_CNT3), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA3_EDC_CNT3), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA4_EDC_CNT3), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA5_EDC_CNT3), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA6_EDC_CNT3), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT2), 0, 0, 0 }, + { SOC15_REG_ENTRY(MMHUB, 0, mmMMEA7_EDC_CNT3), 0, 0, 0 }, }; static int mmhub_v9_4_get_ras_error_count(const struct soc15_reg_entry *reg, diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c index f737ce459c28..cf557a428298 100644 --- a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c @@ -426,7 +426,7 @@ static int navi10_ih_set_clockgating_state(void *handle, struct amdgpu_device *adev = (struct amdgpu_device *)handle; navi10_ih_update_clockgating_state(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index 2e0f8933410e..2d1bebdf1603 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -950,13 +950,13 @@ static int nv_common_set_clockgating_state(void *handle, case CHIP_NAVI14: case CHIP_NAVI12: adev->nbio.funcs->update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); nv_update_hdp_mem_power_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); nv_update_hdp_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index 685dd9754c67..0829188c1a5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -22,6 +22,7 @@ #include <linux/firmware.h> #include <linux/module.h> +#include <linux/vmalloc.h> #include "amdgpu.h" #include "amdgpu_psp.h" @@ -971,10 +972,13 @@ Err_out: */ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops) { - int ret; - uint32_t p2c_header[4]; struct psp_memory_training_context *ctx = &psp->mem_train_ctx; uint32_t *pcache = (uint32_t*)ctx->sys_cache; + struct amdgpu_device *adev = psp->adev; + uint32_t p2c_header[4]; + uint32_t sz; + void *buf; + int ret; if (ctx->init == PSP_MEM_TRAIN_NOT_SUPPORT) { DRM_DEBUG("Memory training is not supported.\n"); @@ -989,7 +993,7 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops) return 0; } - amdgpu_device_vram_access(psp->adev, ctx->p2c_train_data_offset, p2c_header, sizeof(p2c_header), false); + amdgpu_device_vram_access(adev, ctx->p2c_train_data_offset, p2c_header, sizeof(p2c_header), false); DRM_DEBUG("sys_cache[%08x,%08x,%08x,%08x] p2c_header[%08x,%08x,%08x,%08x]\n", pcache[0], pcache[1], pcache[2], pcache[3], p2c_header[0], p2c_header[1], p2c_header[2], p2c_header[3]); @@ -1026,11 +1030,38 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops) DRM_DEBUG("Memory training ops:%x.\n", ops); if (ops & PSP_MEM_TRAIN_SEND_LONG_MSG) { + /* + * Long traing will encroach certain mount of bottom VRAM, + * saving the content of this bottom VRAM to system memory + * before training, and restoring it after training to avoid + * VRAM corruption. + */ + sz = GDDR6_MEM_TRAINING_ENCROACHED_SIZE; + + if (adev->gmc.visible_vram_size < sz || !adev->mman.aper_base_kaddr) { + DRM_ERROR("visible_vram_size %llx or aper_base_kaddr %p is not initialized.\n", + adev->gmc.visible_vram_size, + adev->mman.aper_base_kaddr); + return -EINVAL; + } + + buf = vmalloc(sz); + if (!buf) { + DRM_ERROR("failed to allocate system memory.\n"); + return -ENOMEM; + } + + memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz); ret = psp_v11_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN); if (ret) { DRM_ERROR("Send long training msg failed.\n"); + vfree(buf); return ret; } + + memcpy_toio(adev->mman.aper_base_kaddr, buf, sz); + adev->nbio.funcs->hdp_flush(adev, NULL); + vfree(buf); } if (ops & PSP_MEM_TRAIN_SAVE) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index 27c7001be1ee..e55884d204bd 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -2176,9 +2176,9 @@ static int sdma_v4_0_set_clockgating_state(void *handle, case CHIP_ARCTURUS: case CHIP_RENOIR: sdma_v4_0_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); sdma_v4_0_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 4c6bf1f8a528..67b9830b7c7e 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -1525,9 +1525,9 @@ static int sdma_v5_0_set_clockgating_state(void *handle, case CHIP_NAVI14: case CHIP_NAVI12: sdma_v5_0_update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); sdma_v5_0_update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c index 9aac9f9c50bb..42d5601b6bf3 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dma.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c @@ -648,7 +648,7 @@ static int si_dma_set_clockgating_state(void *handle, bool enable; struct amdgpu_device *adev = (struct amdgpu_device *)handle; - enable = (state == AMD_CG_STATE_GATE) ? true : false; + enable = (state == AMD_CG_STATE_GATE); if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) { for (i = 0; i < adev->sdma.num_instances; i++) { diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 317803f6a561..15f3424a1ff7 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -537,6 +537,10 @@ soc15_asic_reset_method(struct amdgpu_device *adev) static int soc15_asic_reset(struct amdgpu_device *adev) { + /* original raven doesn't have full asic reset */ + if (adev->pdev->device == 0x15dd && adev->rev_id < 0x8) + return 0; + switch (soc15_asic_reset_method(adev)) { case AMD_RESET_METHOD_BACO: if (!adev->in_suspend) @@ -1467,38 +1471,38 @@ static int soc15_common_set_clockgating_state(void *handle, case CHIP_VEGA12: case CHIP_VEGA20: adev->nbio.funcs->update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_hdp_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_drm_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_drm_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_rom_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); adev->df.funcs->update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; case CHIP_RAVEN: case CHIP_RENOIR: adev->nbio.funcs->update_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_hdp_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_drm_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_drm_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); soc15_update_rom_medium_grain_clock_gating(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; case CHIP_ARCTURUS: soc15_update_hdp_light_sleep(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c index 01e62fb8e6e0..0fa8aae2d78e 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c @@ -763,7 +763,7 @@ static int uvd_v5_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); if (enable) { /* wait for STATUS to clear */ diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c index 217084d56ab8..e0aadcaf6c8b 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c @@ -1421,7 +1421,7 @@ static int uvd_v6_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); if (enable) { /* wait for STATUS to clear */ diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index 475ae68f38f5..217db187207c 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -739,7 +739,7 @@ static int vce_v3_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); int i; if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)) diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c index 683701cf7270..3fd102efb7af 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c @@ -887,7 +887,7 @@ static int vce_v4_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); int i; if ((adev->asic_type == CHIP_POLARIS10) || diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c index e654938f6cca..1a24fadd30e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c @@ -1346,7 +1346,7 @@ static int vcn_v1_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); if (enable) { /* wait for STATUS to clear */ diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c index f4db8af6536b..4f7216788f11 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c @@ -1213,7 +1213,7 @@ static int vcn_v2_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); if (enable) { /* wait for STATUS to clear */ @@ -1624,7 +1624,7 @@ static int vcn_v2_0_process_interrupt(struct amdgpu_device *adev, return 0; } -static int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring) +int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; uint32_t tmp = 0; diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h index ef749b02ded9..6c9de1882428 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h @@ -37,6 +37,7 @@ extern void vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vmid, uint64_t pd_addr); extern void vcn_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val); +extern int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring); extern void vcn_v2_0_enc_ring_insert_end(struct amdgpu_ring *ring); extern void vcn_v2_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c index c8b63d57a541..70fae7977f8f 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c @@ -435,88 +435,88 @@ static void vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { if (!indirect) { WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), + UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), + UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); } else { WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); + UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); + UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); } offset = 0; } else { WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), + UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), + UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); offset = size; WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), + UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); } if (!indirect) WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect); else WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); /* cache window 1: stack */ if (!indirect) { WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), + UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), + UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); } else { WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); + UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); + UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); } WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); /* cache window 2: context */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), + UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), + UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); + UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); /* non-cache window */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 0, 0, indirect); + UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 0, 0, indirect); + UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0), 0, 0, indirect); + UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0), 0, 0, indirect); /* VCN global tiling registers */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); + UVD, 0, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); } /** @@ -670,19 +670,19 @@ static void vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_device *adev, UVD_CGC_CTRL__VCPU_MODE_MASK | UVD_CGC_CTRL__MMSCH_MODE_MASK); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect); + UVD, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect); /* turn off clock gating */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect); + UVD, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect); /* turn on SUVD clock gating */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); + UVD, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); /* turn on sw mode in UVD_SUVD_CGC_CTRL */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); + UVD, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); } /** @@ -772,11 +772,11 @@ static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, boo tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect); + UVD, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect); /* disable master interupt */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect); + UVD, 0, mmUVD_MASTINT_EN), 0, 0, indirect); /* setup mmUVD_LMI_CTRL */ tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | @@ -788,28 +788,28 @@ static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, boo (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | 0x00100000L); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect); + UVD, 0, mmUVD_LMI_CTRL), tmp, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_MPC_CNTL), + UVD, 0, mmUVD_MPC_CNTL), 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_MPC_SET_MUXA0), + UVD, 0, mmUVD_MPC_SET_MUXA0), ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_MPC_SET_MUXB0), + UVD, 0, mmUVD_MPC_SET_MUXB0), ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_MPC_SET_MUX), + UVD, 0, mmUVD_MPC_SET_MUX), ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); @@ -817,26 +817,26 @@ static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, boo vcn_v2_5_mc_resume_dpg_mode(adev, inst_idx, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect); + UVD, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect); WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect); + UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect); /* enable LMI MC and UMC channels */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect); + UVD, 0, mmUVD_LMI_CTRL2), 0, 0, indirect); /* unblock VCPU register access */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect); + UVD, 0, mmUVD_RB_ARB_CTRL), 0, 0, indirect); tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect); + UVD, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect); /* enable master interrupt */ WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0( - UVD, inst_idx, mmUVD_MASTINT_EN), + UVD, 0, mmUVD_MASTINT_EN), UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); if (indirect) @@ -891,8 +891,10 @@ static int vcn_v2_5_start(struct amdgpu_device *adev) for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { if (adev->vcn.harvest_config & (1 << i)) continue; - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) - return vcn_v2_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram); + if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { + r = vcn_v2_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram); + continue; + } /* disable register anti-hang mechanism */ WREG32_P(SOC15_REG_OFFSET(UVD, i, mmUVD_POWER_STATUS), 0, @@ -903,6 +905,9 @@ static int vcn_v2_5_start(struct amdgpu_device *adev) WREG32_SOC15(UVD, i, mmUVD_STATUS, tmp); } + if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) + return 0; + /*SW clock gating */ vcn_v2_5_disable_clock_gating(adev); @@ -1294,10 +1299,9 @@ static int vcn_v2_5_stop(struct amdgpu_device *adev) for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { if (adev->vcn.harvest_config & (1 << i)) continue; - if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { r = vcn_v2_5_stop_dpg_mode(adev, i); - goto power_off; + continue; } /* wait for vcn idle */ @@ -1349,7 +1353,6 @@ static int vcn_v2_5_stop(struct amdgpu_device *adev) ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); } -power_off: if (adev->pm.dpm_enabled) amdgpu_dpm_enable_uvd(adev, false); @@ -1488,7 +1491,7 @@ static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = { .emit_ib = vcn_v2_0_dec_ring_emit_ib, .emit_fence = vcn_v2_0_dec_ring_emit_fence, .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush, - .test_ring = amdgpu_vcn_dec_ring_test_ring, + .test_ring = vcn_v2_0_dec_ring_test_ring, .test_ib = amdgpu_vcn_dec_ring_test_ib, .insert_nop = vcn_v2_0_dec_ring_insert_nop, .insert_start = vcn_v2_0_dec_ring_insert_start, @@ -1663,7 +1666,7 @@ static int vcn_v2_5_set_clockgating_state(void *handle, enum amd_clockgating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - bool enable = (state == AMD_CG_STATE_GATE) ? true : false; + bool enable = (state == AMD_CG_STATE_GATE); if (amdgpu_sriov_vf(adev)) return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c index d9e331084ea0..407c6093c2ec 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c @@ -717,7 +717,7 @@ static int vega10_ih_set_clockgating_state(void *handle, struct amdgpu_device *adev = (struct amdgpu_device *)handle; vega10_ih_update_clockgating_state(adev, - state == AMD_CG_STATE_GATE ? true : false); + state == AMD_CG_STATE_GATE); return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 2870553a2ce0..80d22bf702e8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -1237,16 +1237,18 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, list_add(&q->list, &qpd->queues_list); qpd->queue_count++; + + if (q->properties.type == KFD_QUEUE_TYPE_SDMA) + dqm->sdma_queue_count++; + else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + dqm->xgmi_sdma_queue_count++; + if (q->properties.is_active) { dqm->queue_count++; retval = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0); } - if (q->properties.type == KFD_QUEUE_TYPE_SDMA) - dqm->sdma_queue_count++; - else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) - dqm->xgmi_sdma_queue_count++; /* * Unconditionally increment this counter, regardless of the queue's * type or whether the queue is active. diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 9402374d2466..279541517a99 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -940,14 +940,14 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) goto error; } - dc_hardware_init(adev->dm.dc); - r = dm_dmub_hw_init(adev); if (r) { DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); goto error; } + dc_hardware_init(adev->dm.dc); + adev->dm.freesync_module = mod_freesync_create(adev->dm.dc); if (!adev->dm.freesync_module) { DRM_ERROR( @@ -7759,24 +7759,27 @@ dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm, struct drm_crtc_state *new_crtc_state, *old_crtc_state; struct dm_crtc_state *new_dm_crtc_state, *old_dm_crtc_state; struct dc_stream_status *status = NULL; - - struct dc_surface_update *updates; enum surface_update_type update_type = UPDATE_TYPE_FAST; + struct surface_info_bundle { + struct dc_surface_update surface_updates[MAX_SURFACES]; + struct dc_plane_info plane_infos[MAX_SURFACES]; + struct dc_scaling_info scaling_infos[MAX_SURFACES]; + struct dc_flip_addrs flip_addrs[MAX_SURFACES]; + struct dc_stream_update stream_update; + } *bundle; - updates = kcalloc(MAX_SURFACES, sizeof(*updates), GFP_KERNEL); + bundle = kzalloc(sizeof(*bundle), GFP_KERNEL); - if (!updates) { - DRM_ERROR("Failed to allocate plane updates\n"); + if (!bundle) { + DRM_ERROR("Failed to allocate update bundle\n"); /* Set type to FULL to avoid crashing in DC*/ update_type = UPDATE_TYPE_FULL; goto cleanup; } for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - struct dc_scaling_info scaling_info; - struct dc_stream_update stream_update; - memset(&stream_update, 0, sizeof(stream_update)); + memset(bundle, 0, sizeof(struct surface_info_bundle)); new_dm_crtc_state = to_dm_crtc_state(new_crtc_state); old_dm_crtc_state = to_dm_crtc_state(old_crtc_state); @@ -7793,8 +7796,9 @@ dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm, for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, j) { const struct amdgpu_framebuffer *amdgpu_fb = to_amdgpu_framebuffer(new_plane_state->fb); - struct dc_plane_info plane_info; - struct dc_flip_addrs flip_addr; + struct dc_plane_info *plane_info = &bundle->plane_infos[num_plane]; + struct dc_flip_addrs *flip_addr = &bundle->flip_addrs[num_plane]; + struct dc_scaling_info *scaling_info = &bundle->scaling_infos[num_plane]; uint64_t tiling_flags; new_plane_crtc = new_plane_state->crtc; @@ -7812,49 +7816,48 @@ dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm, if (crtc != new_plane_crtc) continue; - updates[num_plane].surface = new_dm_plane_state->dc_state; + bundle->surface_updates[num_plane].surface = + new_dm_plane_state->dc_state; if (new_crtc_state->mode_changed) { - stream_update.dst = new_dm_crtc_state->stream->dst; - stream_update.src = new_dm_crtc_state->stream->src; + bundle->stream_update.dst = new_dm_crtc_state->stream->dst; + bundle->stream_update.src = new_dm_crtc_state->stream->src; } if (new_crtc_state->color_mgmt_changed) { - updates[num_plane].gamma = + bundle->surface_updates[num_plane].gamma = new_dm_plane_state->dc_state->gamma_correction; - updates[num_plane].in_transfer_func = + bundle->surface_updates[num_plane].in_transfer_func = new_dm_plane_state->dc_state->in_transfer_func; - stream_update.gamut_remap = + bundle->stream_update.gamut_remap = &new_dm_crtc_state->stream->gamut_remap_matrix; - stream_update.output_csc_transform = + bundle->stream_update.output_csc_transform = &new_dm_crtc_state->stream->csc_color_matrix; - stream_update.out_transfer_func = + bundle->stream_update.out_transfer_func = new_dm_crtc_state->stream->out_transfer_func; } ret = fill_dc_scaling_info(new_plane_state, - &scaling_info); + scaling_info); if (ret) goto cleanup; - updates[num_plane].scaling_info = &scaling_info; + bundle->surface_updates[num_plane].scaling_info = scaling_info; if (amdgpu_fb) { ret = get_fb_info(amdgpu_fb, &tiling_flags); if (ret) goto cleanup; - memset(&flip_addr, 0, sizeof(flip_addr)); - ret = fill_dc_plane_info_and_addr( dm->adev, new_plane_state, tiling_flags, - &plane_info, - &flip_addr.address); + plane_info, + &flip_addr->address); if (ret) goto cleanup; - updates[num_plane].plane_info = &plane_info; - updates[num_plane].flip_addr = &flip_addr; + bundle->surface_updates[num_plane].plane_info = plane_info; + bundle->surface_updates[num_plane].flip_addr = flip_addr; } num_plane++; @@ -7875,14 +7878,15 @@ dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm, status = dc_stream_get_status_from_state(old_dm_state->context, new_dm_crtc_state->stream); - stream_update.stream = new_dm_crtc_state->stream; + bundle->stream_update.stream = new_dm_crtc_state->stream; /* * TODO: DC modifies the surface during this call so we need * to lock here - find a way to do this without locking. */ mutex_lock(&dm->dc_lock); - update_type = dc_check_update_surfaces_for_stream(dc, updates, num_plane, - &stream_update, status); + update_type = dc_check_update_surfaces_for_stream( + dc, bundle->surface_updates, num_plane, + &bundle->stream_update, status); mutex_unlock(&dm->dc_lock); if (update_type > UPDATE_TYPE_MED) { @@ -7892,7 +7896,7 @@ dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm, } cleanup: - kfree(updates); + kfree(bundle); *out_type = update_type; return ret; @@ -8163,6 +8167,16 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, goto fail; #endif + /* + * Perform validation of MST topology in the state: + * We need to perform MST atomic check before calling + * dc_validate_global_state(), or there is a chance + * to get stuck in an infinite loop and hang eventually. + */ + ret = drm_dp_mst_atomic_check(state); + if (ret) + goto fail; + if (dc_validate_global_state(dc, dm_state->context, false) != DC_OK) { ret = -EINVAL; goto fail; @@ -8191,10 +8205,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, dc_retain_state(old_dm_state->context); } } - /* Perform validation of MST topology in the state*/ - ret = drm_dp_mst_atomic_check(state); - if (ret) - goto fail; /* Store the overall update type for use later in atomic check. */ for_each_new_crtc_in_state (state, crtc, new_crtc_state, i) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index ae329335dfcc..0acd3409dd6c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -135,6 +135,20 @@ void hdcp_update_display(struct hdcp_workqueue *hdcp_work, mutex_unlock(&hdcp_w->mutex); } +static void hdcp_remove_display(struct hdcp_workqueue *hdcp_work, + unsigned int link_index, + struct amdgpu_dm_connector *aconnector) +{ + struct hdcp_workqueue *hdcp_w = &hdcp_work[link_index]; + + mutex_lock(&hdcp_w->mutex); + hdcp_w->aconnector = aconnector; + + mod_hdcp_remove_display(&hdcp_w->hdcp, aconnector->base.index, &hdcp_w->output); + + process_output(hdcp_w); + mutex_unlock(&hdcp_w->mutex); +} void hdcp_reset_display(struct hdcp_workqueue *hdcp_work, unsigned int link_index) { struct hdcp_workqueue *hdcp_w = &hdcp_work[link_index]; @@ -303,6 +317,11 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) memset(link, 0, sizeof(*link)); display->index = aconnector->base.index; + + if (config->dpms_off) { + hdcp_remove_display(hdcp_work, link_index, aconnector); + return; + } display->state = MOD_HDCP_DISPLAY_ACTIVE; if (aconnector->dc_sink != NULL) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index 069b7a6f5597..318b474ff20e 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -216,7 +216,8 @@ bool dm_helpers_dp_mst_write_payload_allocation_table( drm_dp_mst_reset_vcpi_slots(mst_mgr, mst_port); } - ret = drm_dp_update_payload_part1(mst_mgr); + /* It's OK for this to fail */ + drm_dp_update_payload_part1(mst_mgr); /* mst_mgr->->payloads are VC payload notify MST branch using DPCD or * AUX message. The sequence is slot 1-63 allocated sequence for each @@ -225,9 +226,6 @@ bool dm_helpers_dp_mst_write_payload_allocation_table( get_payload_table(aconnector, proposed_table); - if (ret) - return false; - return true; } @@ -285,7 +283,6 @@ bool dm_helpers_dp_mst_send_payload_allocation( struct amdgpu_dm_connector *aconnector; struct drm_dp_mst_topology_mgr *mst_mgr; struct drm_dp_mst_port *mst_port; - int ret; aconnector = (struct amdgpu_dm_connector *)stream->dm_stream_context; @@ -299,10 +296,8 @@ bool dm_helpers_dp_mst_send_payload_allocation( if (!mst_mgr->mst_state) return false; - ret = drm_dp_update_payload_part2(mst_mgr); - - if (ret) - return false; + /* It's OK for this to fail */ + drm_dp_update_payload_part2(mst_mgr); if (!enable) drm_dp_mst_deallocate_vcpi(mst_mgr, mst_port); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 96b391e4b3e7..5672f7765919 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -632,7 +632,7 @@ static void increase_dsc_bpp(struct drm_atomic_state *state, if (drm_dp_atomic_find_vcpi_slots(state, params[next_index].port->mgr, params[next_index].port, - vars[next_index].pbn,\ + vars[next_index].pbn, dm_mst_get_pbn_divider(dc_link)) < 0) return; if (!drm_dp_mst_atomic_check(state)) { diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c index 2cb7a4288cb7..629a07a2719b 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c @@ -89,6 +89,10 @@ static enum bp_result encoder_control_digx_v1_5( struct bios_parser *bp, struct bp_encoder_control *cntl); +static enum bp_result encoder_control_fallback( + struct bios_parser *bp, + struct bp_encoder_control *cntl); + static void init_dig_encoder_control(struct bios_parser *bp) { uint32_t version = @@ -100,7 +104,7 @@ static void init_dig_encoder_control(struct bios_parser *bp) break; default: dm_output_to_console("Don't have dig_encoder_control for v%d\n", version); - bp->cmd_tbl.dig_encoder_control = NULL; + bp->cmd_tbl.dig_encoder_control = encoder_control_fallback; break; } } @@ -184,6 +188,18 @@ static enum bp_result encoder_control_digx_v1_5( return result; } +static enum bp_result encoder_control_fallback( + struct bios_parser *bp, + struct bp_encoder_control *cntl) +{ + if (bp->base.ctx->dc->ctx->dmub_srv && + bp->base.ctx->dc->debug.dmub_command_table) { + return encoder_control_digx_v1_5(bp, cntl); + } + + return BP_RESULT_FAILURE; +} + /***************************************************************************** ****************************************************************************** ** @@ -196,6 +212,10 @@ static enum bp_result transmitter_control_v1_6( struct bios_parser *bp, struct bp_transmitter_control *cntl); +static enum bp_result transmitter_control_fallback( + struct bios_parser *bp, + struct bp_transmitter_control *cntl); + static void init_transmitter_control(struct bios_parser *bp) { uint8_t frev; @@ -209,7 +229,7 @@ static void init_transmitter_control(struct bios_parser *bp) break; default: dm_output_to_console("Don't have transmitter_control for v%d\n", crev); - bp->cmd_tbl.transmitter_control = NULL; + bp->cmd_tbl.transmitter_control = transmitter_control_fallback; break; } } @@ -273,6 +293,18 @@ static enum bp_result transmitter_control_v1_6( return result; } +static enum bp_result transmitter_control_fallback( + struct bios_parser *bp, + struct bp_transmitter_control *cntl) +{ + if (bp->base.ctx->dc->ctx->dmub_srv && + bp->base.ctx->dc->debug.dmub_command_table) { + return transmitter_control_v1_6(bp, cntl); + } + + return BP_RESULT_FAILURE; +} + /****************************************************************************** ****************************************************************************** ** @@ -285,6 +317,10 @@ static enum bp_result set_pixel_clock_v7( struct bios_parser *bp, struct bp_pixel_clock_parameters *bp_params); +static enum bp_result set_pixel_clock_fallback( + struct bios_parser *bp, + struct bp_pixel_clock_parameters *bp_params); + static void init_set_pixel_clock(struct bios_parser *bp) { switch (BIOS_CMD_TABLE_PARA_REVISION(setpixelclock)) { @@ -294,7 +330,7 @@ static void init_set_pixel_clock(struct bios_parser *bp) default: dm_output_to_console("Don't have set_pixel_clock for v%d\n", BIOS_CMD_TABLE_PARA_REVISION(setpixelclock)); - bp->cmd_tbl.set_pixel_clock = NULL; + bp->cmd_tbl.set_pixel_clock = set_pixel_clock_fallback; break; } } @@ -400,6 +436,18 @@ static enum bp_result set_pixel_clock_v7( return result; } +static enum bp_result set_pixel_clock_fallback( + struct bios_parser *bp, + struct bp_pixel_clock_parameters *bp_params) +{ + if (bp->base.ctx->dc->ctx->dmub_srv && + bp->base.ctx->dc->debug.dmub_command_table) { + return set_pixel_clock_v7(bp, bp_params); + } + + return BP_RESULT_FAILURE; +} + /****************************************************************************** ****************************************************************************** ** @@ -632,6 +680,11 @@ static enum bp_result enable_disp_power_gating_v2_1( enum controller_id crtc_id, enum bp_pipe_control_action action); +static enum bp_result enable_disp_power_gating_fallback( + struct bios_parser *bp, + enum controller_id crtc_id, + enum bp_pipe_control_action action); + static void init_enable_disp_power_gating( struct bios_parser *bp) { @@ -643,7 +696,7 @@ static void init_enable_disp_power_gating( default: dm_output_to_console("Don't enable_disp_power_gating enable_crtc for v%d\n", BIOS_CMD_TABLE_PARA_REVISION(enabledisppowergating)); - bp->cmd_tbl.enable_disp_power_gating = NULL; + bp->cmd_tbl.enable_disp_power_gating = enable_disp_power_gating_fallback; break; } } @@ -658,6 +711,10 @@ static void enable_disp_power_gating_dmcub( power_gating.header.sub_type = DMUB_CMD__VBIOS_ENABLE_DISP_POWER_GATING; power_gating.power_gating.pwr = *pwr; + /* ATOM_ENABLE is old API in DMUB */ + if (power_gating.power_gating.pwr.enable == ATOM_ENABLE) + power_gating.power_gating.pwr.enable = ATOM_INIT; + dc_dmub_srv_cmd_queue(dmcub, &power_gating.header); dc_dmub_srv_cmd_execute(dmcub); dc_dmub_srv_wait_idle(dmcub); @@ -695,6 +752,19 @@ static enum bp_result enable_disp_power_gating_v2_1( return result; } +static enum bp_result enable_disp_power_gating_fallback( + struct bios_parser *bp, + enum controller_id crtc_id, + enum bp_pipe_control_action action) +{ + if (bp->base.ctx->dc->ctx->dmub_srv && + bp->base.ctx->dc->debug.dmub_command_table) { + return enable_disp_power_gating_v2_1(bp, crtc_id, action); + } + + return BP_RESULT_FAILURE; +} + /****************************************************************************** ******************************************************************************* ** diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c index a27d84ca15a5..1a37550731de 100644 --- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c +++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c @@ -1435,6 +1435,7 @@ void dcn_bw_update_from_pplib(struct dc *dc) struct dc_context *ctx = dc->ctx; struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0}; bool res; + unsigned vmin0p65_idx, vmid0p72_idx, vnom0p8_idx, vmax0p9_idx; /* TODO: This is not the proper way to obtain fabric_and_dram_bandwidth, should be min(fclk, memclk) */ res = dm_pp_get_clock_levels_by_type_with_voltage( @@ -1446,17 +1447,28 @@ void dcn_bw_update_from_pplib(struct dc *dc) res = verify_clock_values(&fclks); if (res) { - ASSERT(fclks.num_levels >= 3); - dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = 32 * (fclks.data[0].clocks_in_khz / 1000.0) / 1000.0; - dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = dc->dcn_soc->number_of_channels * - (fclks.data[fclks.num_levels - (fclks.num_levels > 2 ? 3 : 2)].clocks_in_khz / 1000.0) - * ddr4_dram_factor_single_Channel / 1000.0; - dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = dc->dcn_soc->number_of_channels * - (fclks.data[fclks.num_levels - 2].clocks_in_khz / 1000.0) - * ddr4_dram_factor_single_Channel / 1000.0; - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = dc->dcn_soc->number_of_channels * - (fclks.data[fclks.num_levels - 1].clocks_in_khz / 1000.0) - * ddr4_dram_factor_single_Channel / 1000.0; + ASSERT(fclks.num_levels); + + vmin0p65_idx = 0; + vmid0p72_idx = fclks.num_levels - + (fclks.num_levels > 2 ? 3 : (fclks.num_levels > 1 ? 2 : 1)); + vnom0p8_idx = fclks.num_levels - (fclks.num_levels > 1 ? 2 : 1); + vmax0p9_idx = fclks.num_levels - 1; + + dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = + 32 * (fclks.data[vmin0p65_idx].clocks_in_khz / 1000.0) / 1000.0; + dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = + dc->dcn_soc->number_of_channels * + (fclks.data[vmid0p72_idx].clocks_in_khz / 1000.0) + * ddr4_dram_factor_single_Channel / 1000.0; + dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = + dc->dcn_soc->number_of_channels * + (fclks.data[vnom0p8_idx].clocks_in_khz / 1000.0) + * ddr4_dram_factor_single_Channel / 1000.0; + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = + dc->dcn_soc->number_of_channels * + (fclks.data[vmax0p9_idx].clocks_in_khz / 1000.0) + * ddr4_dram_factor_single_Channel / 1000.0; } else BREAK_TO_DEBUGGER(); diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 6c797fac189d..04441dbcba76 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -2462,12 +2462,7 @@ void dc_set_power_state( enum dc_acpi_cm_power_state power_state) { struct kref refcount; - struct display_mode_lib *dml = kzalloc(sizeof(struct display_mode_lib), - GFP_KERNEL); - - ASSERT(dml); - if (!dml) - return; + struct display_mode_lib *dml; switch (power_state) { case DC_ACPI_CM_POWER_STATE_D0: @@ -2490,6 +2485,12 @@ void dc_set_power_state( * clean state, and dc hw programming optimizations will not * cause any trouble. */ + dml = kzalloc(sizeof(struct display_mode_lib), + GFP_KERNEL); + + ASSERT(dml); + if (!dml) + return; /* Preserve refcount */ refcount = dc->current_state->refcount; @@ -2503,10 +2504,10 @@ void dc_set_power_state( dc->current_state->refcount = refcount; dc->current_state->bw_ctx.dml = *dml; + kfree(dml); + break; } - - kfree(dml); } void dc_resume(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index 260c0b62d37d..a09119c10d7c 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -851,18 +851,12 @@ static bool dc_link_detect_helper(struct dc_link *link, if (memcmp(&link->dpcd_caps, &prev_dpcd_caps, sizeof(struct dpcd_caps))) same_dpcd = false; } - /* Active dongle plug in without display or downstream unplug*/ + /* Active dongle downstream unplug*/ if (link->type == dc_connection_active_dongle && link->dpcd_caps.sink_count.bits.SINK_COUNT == 0) { - if (prev_sink != NULL) { + if (prev_sink != NULL) /* Downstream unplug */ dc_sink_release(prev_sink); - } else { - /* Empty dongle plug in */ - dp_verify_link_cap_with_retries(link, - &link->reported_link_cap, - LINK_TRAINING_MAX_VERIFY_RETRY); - } return true; } @@ -969,8 +963,7 @@ static bool dc_link_detect_helper(struct dc_link *link, same_edid = is_same_edid(&prev_sink->dc_edid, &sink->dc_edid); if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT && - sink_caps.transaction_type == DDC_TRANSACTION_TYPE_I2C_OVER_AUX && - reason != DETECT_REASON_HPDRX) { + sink_caps.transaction_type == DDC_TRANSACTION_TYPE_I2C_OVER_AUX) { /* * TODO debug why Dell 2413 doesn't like * two link trainings @@ -2882,7 +2875,16 @@ enum dc_status dc_link_reallocate_mst_payload(struct dc_link *link) // Clear all of MST payload then reallocate for (i = 0; i < MAX_PIPES; i++) { pipe_ctx = &link->dc->current_state->res_ctx.pipe_ctx[i]; - if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->link == link && + + /* driver enable split pipe for external monitors + * we have to check pipe_ctx is split pipe or not + * If it's split pipe, driver using top pipe to + * reaallocate. + */ + if (!pipe_ctx || pipe_ctx->top_pipe) + continue; + + if (pipe_ctx->stream && pipe_ctx->stream->link == link && pipe_ctx->stream->dpms_off == false && pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { deallocate_mst_payload(pipe_ctx); @@ -2891,7 +2893,11 @@ enum dc_status dc_link_reallocate_mst_payload(struct dc_link *link) for (i = 0; i < MAX_PIPES; i++) { pipe_ctx = &link->dc->current_state->res_ctx.pipe_ctx[i]; - if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->link == link && + + if (!pipe_ctx || pipe_ctx->top_pipe) + continue; + + if (pipe_ctx->stream && pipe_ctx->stream->link == link && pipe_ctx->stream->dpms_off == false && pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { /* enable/disable PHY will clear connection between BE and FE diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index 6ab298c65247..cb731c1d30b1 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -983,7 +983,7 @@ static enum link_training_result perform_clock_recovery_sequence( offset); /* 2. update DPCD of the receiver*/ - if (!retries_cr) + if (!retry_count) /* EPR #361076 - write as a 5-byte burst, * but only for the 1-st iteration.*/ dpcd_set_lt_pattern_and_lane_settings( @@ -3680,7 +3680,7 @@ static void set_crtc_test_pattern(struct dc_link *link, struct pipe_ctx *odm_pipe; enum controller_dp_color_space controller_color_space; int opp_cnt = 1; - uint8_t count = 0; + int count; switch (test_pattern_color_space) { case DP_TEST_PATTERN_COLOR_SPACE_RGB: @@ -3725,11 +3725,11 @@ static void set_crtc_test_pattern(struct dc_link *link, width, height); /* wait for dpg to blank pixel data with test pattern */ - for (count = 0; count < 1000; count++) + for (count = 0; count < 1000; count++) { if (opp->funcs->dpg_is_blanked(opp)) break; - else - udelay(100); + udelay(100); + } } } break; @@ -3925,8 +3925,38 @@ bool dc_link_dp_set_test_pattern( sizeof(training_pattern)); } } else { - /* CRTC Patterns */ + enum dc_color_space color_space = COLOR_SPACE_UNKNOWN; + + switch (test_pattern_color_space) { + case DP_TEST_PATTERN_COLOR_SPACE_RGB: + color_space = COLOR_SPACE_SRGB; + if (test_pattern == DP_TEST_PATTERN_COLOR_SQUARES_CEA) + color_space = COLOR_SPACE_SRGB_LIMITED; + break; + + case DP_TEST_PATTERN_COLOR_SPACE_YCBCR601: + color_space = COLOR_SPACE_YCBCR601; + if (test_pattern == DP_TEST_PATTERN_COLOR_SQUARES_CEA) + color_space = COLOR_SPACE_YCBCR601_LIMITED; + break; + case DP_TEST_PATTERN_COLOR_SPACE_YCBCR709: + color_space = COLOR_SPACE_YCBCR709; + if (test_pattern == DP_TEST_PATTERN_COLOR_SQUARES_CEA) + color_space = COLOR_SPACE_YCBCR709_LIMITED; + break; + default: + break; + } + /* update MSA to requested color space */ + pipe_ctx->stream_res.stream_enc->funcs->dp_set_stream_attribute(pipe_ctx->stream_res.stream_enc, + &pipe_ctx->stream->timing, + color_space, + pipe_ctx->stream->use_vsc_sdp_for_colorimetry, + link->dpcd_caps.dprx_feature.bits.SST_SPLIT_SDP_CAP); + + /* CRTC Patterns */ set_crtc_test_pattern(link, pipe_ctx, test_pattern, test_pattern_color_space); + /* Set Test Pattern state */ link->test_pattern_enabled = true; } diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 3fa85a54360f..8ff25b5dd2f6 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -39,7 +39,7 @@ #include "inc/hw/dmcu.h" #include "dml/display_mode_lib.h" -#define DC_VER "3.2.68" +#define DC_VER "3.2.69" #define MAX_SURFACES 3 #define MAX_PLANES 6 @@ -425,6 +425,7 @@ struct dc_debug_options { bool validate_dml_output; bool enable_dmcub_surface_flip; bool usbc_combo_phy_reset_wa; + bool disable_dsc; }; struct dc_debug_data { diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c index 1cd4d8fc361f..066188ba7949 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c @@ -100,20 +100,6 @@ static uint32_t get_hw_buffer_available_size( dce_i2c_hw->buffer_used_bytes; } -static uint32_t get_speed( - const struct dce_i2c_hw *dce_i2c_hw) -{ - uint32_t pre_scale = 0; - - REG_GET(SPEED, DC_I2C_DDC1_PRESCALE, &pre_scale); - - /* [anaumov] it seems following is unnecessary */ - /*ASSERT(value.bits.DC_I2C_DDC1_PRESCALE);*/ - return pre_scale ? - dce_i2c_hw->reference_frequency / pre_scale : - dce_i2c_hw->default_speed; -} - static void process_channel_reply( struct dce_i2c_hw *dce_i2c_hw, struct i2c_payload *reply) @@ -278,16 +264,25 @@ static void set_speed( struct dce_i2c_hw *dce_i2c_hw, uint32_t speed) { + uint32_t xtal_ref_div = 0; + uint32_t prescale = 0; + + REG_GET(MICROSECOND_TIME_BASE_DIV, XTAL_REF_DIV, &xtal_ref_div); + + if (xtal_ref_div == 0) + xtal_ref_div = 2; + + prescale = ((dce_i2c_hw->reference_frequency * 2) / xtal_ref_div) / speed; if (speed) { if (dce_i2c_hw->masks->DC_I2C_DDC1_START_STOP_TIMING_CNTL) REG_UPDATE_N(SPEED, 3, - FN(DC_I2C_DDC1_SPEED, DC_I2C_DDC1_PRESCALE), dce_i2c_hw->reference_frequency / speed, + FN(DC_I2C_DDC1_SPEED, DC_I2C_DDC1_PRESCALE), prescale, FN(DC_I2C_DDC1_SPEED, DC_I2C_DDC1_THRESHOLD), 2, FN(DC_I2C_DDC1_SPEED, DC_I2C_DDC1_START_STOP_TIMING_CNTL), speed > 50 ? 2:1); else REG_UPDATE_N(SPEED, 2, - FN(DC_I2C_DDC1_SPEED, DC_I2C_DDC1_PRESCALE), dce_i2c_hw->reference_frequency / speed, + FN(DC_I2C_DDC1_SPEED, DC_I2C_DDC1_PRESCALE), prescale, FN(DC_I2C_DDC1_SPEED, DC_I2C_DDC1_THRESHOLD), 2); } } @@ -344,9 +339,7 @@ static void release_engine( bool safe_to_reset; /* Restore original HW engine speed */ - - set_speed(dce_i2c_hw, dce_i2c_hw->original_speed); - + set_speed(dce_i2c_hw, dce_i2c_hw->default_speed); /* Reset HW engine */ { @@ -378,7 +371,6 @@ struct dce_i2c_hw *acquire_i2c_hw_engine( { uint32_t counter = 0; enum gpio_result result; - uint32_t current_speed; struct dce_i2c_hw *dce_i2c_hw = NULL; if (!ddc) @@ -416,11 +408,6 @@ struct dce_i2c_hw *acquire_i2c_hw_engine( dce_i2c_hw->ddc = ddc; - current_speed = get_speed(dce_i2c_hw); - - if (current_speed) - dce_i2c_hw->original_speed = current_speed; - if (!setup_engine(dce_i2c_hw)) { release_engine(dce_i2c_hw); return NULL; @@ -478,13 +465,9 @@ static void submit_channel_request_hw( static uint32_t get_transaction_timeout_hw( const struct dce_i2c_hw *dce_i2c_hw, - uint32_t length) + uint32_t length, + uint32_t speed) { - - uint32_t speed = get_speed(dce_i2c_hw); - - - uint32_t period_timeout; uint32_t num_of_clock_stretches; @@ -504,7 +487,8 @@ static uint32_t get_transaction_timeout_hw( bool dce_i2c_hw_engine_submit_payload( struct dce_i2c_hw *dce_i2c_hw, struct i2c_payload *payload, - bool middle_of_transaction) + bool middle_of_transaction, + uint32_t speed) { struct i2c_request_transaction_data request; @@ -542,7 +526,7 @@ bool dce_i2c_hw_engine_submit_payload( /* obtain timeout value before submitting request */ transaction_timeout = get_transaction_timeout_hw( - dce_i2c_hw, payload->length + 1); + dce_i2c_hw, payload->length + 1, speed); submit_channel_request_hw( dce_i2c_hw, &request); @@ -588,13 +572,11 @@ bool dce_i2c_submit_command_hw( struct i2c_payload *payload = cmd->payloads + index_of_payload; if (!dce_i2c_hw_engine_submit_payload( - dce_i2c_hw, payload, mot)) { + dce_i2c_hw, payload, mot, cmd->speed)) { result = false; break; } - - ++index_of_payload; } @@ -625,7 +607,6 @@ void dce_i2c_hw_construct( dce_i2c_hw->buffer_used_bytes = 0; dce_i2c_hw->transaction_count = 0; dce_i2c_hw->engine_keep_power_up_count = 1; - dce_i2c_hw->original_speed = DEFAULT_I2C_HW_SPEED; dce_i2c_hw->default_speed = DEFAULT_I2C_HW_SPEED; dce_i2c_hw->send_reset_length = 0; dce_i2c_hw->setup_limit = I2C_SETUP_TIME_LIMIT_DCE; @@ -640,9 +621,6 @@ void dce100_i2c_hw_construct( const struct dce_i2c_shift *shifts, const struct dce_i2c_mask *masks) { - - uint32_t xtal_ref_div = 0; - dce_i2c_hw_construct(dce_i2c_hw, ctx, engine_id, @@ -650,21 +628,6 @@ void dce100_i2c_hw_construct( shifts, masks); dce_i2c_hw->buffer_size = I2C_HW_BUFFER_SIZE_DCE100; - - REG_GET(MICROSECOND_TIME_BASE_DIV, XTAL_REF_DIV, &xtal_ref_div); - - if (xtal_ref_div == 0) - xtal_ref_div = 2; - - /*Calculating Reference Clock by divding original frequency by - * XTAL_REF_DIV. - * At upper level, uint32_t reference_frequency = - * dal_dce_i2c_get_reference_clock(as) >> 1 - * which already divided by 2. So we need x2 to get original - * reference clock from ppll_info - */ - dce_i2c_hw->reference_frequency = - (dce_i2c_hw->reference_frequency * 2) / xtal_ref_div; } void dce112_i2c_hw_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h index d4b2037f7d74..fb055e6883c0 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h @@ -256,7 +256,6 @@ struct i2c_request_transaction_data { struct dce_i2c_hw { struct ddc *ddc; - uint32_t original_speed; uint32_t engine_keep_power_up_count; uint32_t transaction_count; uint32_t buffer_used_bytes; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index f2127afb37b2..1008ac8a0f2a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -2911,6 +2911,33 @@ void dcn10_update_dchub(struct dce_hwseq *hws, struct dchub_init_data *dh_data) hubbub->funcs->update_dchub(hubbub, dh_data); } +static bool dcn10_can_pipe_disable_cursor(struct pipe_ctx *pipe_ctx) +{ + struct pipe_ctx *test_pipe; + const struct rect *r1 = &pipe_ctx->plane_res.scl_data.recout, *r2; + int r1_r = r1->x + r1->width, r1_b = r1->y + r1->height, r2_r, r2_b; + + /** + * Disable the cursor if there's another pipe above this with a + * plane that contains this pipe's viewport to prevent double cursor + * and incorrect scaling artifacts. + */ + for (test_pipe = pipe_ctx->top_pipe; test_pipe; + test_pipe = test_pipe->top_pipe) { + if (!test_pipe->plane_state->visible) + continue; + + r2 = &test_pipe->plane_res.scl_data.recout; + r2_r = r2->x + r2->width; + r2_b = r2->y + r2->height; + + if (r1->x >= r2->x && r1->y >= r2->y && r1_r <= r2_r && r1_b <= r2_b) + return true; + } + + return false; +} + void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx) { struct dc_cursor_position pos_cpy = pipe_ctx->stream->cursor_position; @@ -2956,6 +2983,9 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx) == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE) pos_cpy.enable = false; + if (pos_cpy.enable && dcn10_can_pipe_disable_cursor(pipe_ctx)) + pos_cpy.enable = false; + // Swap axis and mirror horizontally if (param.rotation == ROTATION_ANGLE_90) { uint32_t temp_x = pos_cpy.x; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubp.c index da63fc53cc4a..cf09b9335728 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubp.c @@ -261,28 +261,28 @@ static void hubp21_apply_PLAT_54186_wa( address->video_progressive.luma_addr.high_part == 0xf4) return; - if ((rotation_angle == 0 || rotation_angle == 180) + if ((rotation_angle == ROTATION_ANGLE_0 || rotation_angle == ROTATION_ANGLE_180) && viewport_c_height <= 512) return; - if ((rotation_angle == 90 || rotation_angle == 270) + if ((rotation_angle == ROTATION_ANGLE_90 || rotation_angle == ROTATION_ANGLE_270) && viewport_c_width <= 512) return; switch (rotation_angle) { - case 0: /* 0 degree rotation */ + case ROTATION_ANGLE_0: /* 0 degree rotation */ row_height = 128; patched_viewport_height = (viewport_c_height / row_height + 1) * row_height + 1; patched_viewport_width = viewport_c_width; hubp21->PLAT_54186_wa_chroma_addr_offset = 0; break; - case 2: /* 180 degree rotation */ + case ROTATION_ANGLE_180: /* 180 degree rotation */ row_height = 128; patched_viewport_height = viewport_c_height + row_height; patched_viewport_width = viewport_c_width; hubp21->PLAT_54186_wa_chroma_addr_offset = 0 - chroma_pitch * row_height * chroma_bpe; break; - case 1: /* 90 degree rotation */ + case ROTATION_ANGLE_90: /* 90 degree rotation */ row_height = 256; if (h_mirror_en) { patched_viewport_height = viewport_c_height; @@ -294,7 +294,7 @@ static void hubp21_apply_PLAT_54186_wa( hubp21->PLAT_54186_wa_chroma_addr_offset = 0 - tile_blk_size; } break; - case 3: /* 270 degree rotation */ + case ROTATION_ANGLE_270: /* 270 degree rotation */ row_height = 256; if (h_mirror_en) { patched_viewport_height = viewport_c_height; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index 1d741bca2211..0d506d30d6b6 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -830,7 +830,7 @@ static const struct dc_debug_options debug_defaults_drv = { .disable_dcc = DCC_ENABLE, .vsr_support = true, .performance_trace = false, - .max_downscale_src_width = 3840, + .max_downscale_src_width = 4096, .disable_pplib_wm_range = false, .scl_reset_length10 = true, .sanity_checks = true, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c index e7a8ac7a1f22..45f028986a8d 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c @@ -38,6 +38,7 @@ #define BPP_INVALID 0 #define BPP_BLENDED_PIPE 0xffffffff +#define DCN20_MAX_420_IMAGE_WIDTH 4096 static double adjust_ReturnBW( struct display_mode_lib *mode_lib, @@ -3894,13 +3895,19 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l && i == mode_lib->vba.soc.num_states) mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine = mode_lib->vba.PixelClock[k] / 2 * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0); - if (mode_lib->vba.ODMCapability == false || mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine <= mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity) { - locals->ODMCombineEnablePerState[i][k] = false; - mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine; - } else { - locals->ODMCombineEnablePerState[i][k] = true; - mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + + locals->ODMCombineEnablePerState[i][k] = false; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine; + if (mode_lib->vba.ODMCapability) { + if (locals->PlaneRequiredDISPCLKWithoutODMCombine > mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } else if (locals->HActive[k] > DCN20_MAX_420_IMAGE_WIDTH && locals->OutputFormat[k] == dm_420) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } } + if (locals->MinDPPCLKUsingSingleDPP[k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) <= mode_lib->vba.MaxDppclkRoundedDownToDFSGranularity && locals->SwathWidthYSingleDPP[k] <= locals->MaximumSwathWidth[k] && locals->ODMCombineEnablePerState[i][k] == dm_odm_combine_mode_disabled) { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c index 22f3b5a4b3b9..485a9c62ec58 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c @@ -39,6 +39,7 @@ #define BPP_INVALID 0 #define BPP_BLENDED_PIPE 0xffffffff #define DCN20_MAX_DSC_IMAGE_WIDTH 5184 +#define DCN20_MAX_420_IMAGE_WIDTH 4096 static double adjust_ReturnBW( struct display_mode_lib *mode_lib, @@ -3935,15 +3936,22 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode && i == mode_lib->vba.soc.num_states) mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine = mode_lib->vba.PixelClock[k] / 2 * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0); - if (mode_lib->vba.ODMCapability == false || - (locals->PlaneRequiredDISPCLKWithoutODMCombine <= MaxMaxDispclkRoundedDown - && (!locals->DSCEnabled[k] || locals->HActive[k] <= DCN20_MAX_DSC_IMAGE_WIDTH))) { - locals->ODMCombineEnablePerState[i][k] = false; - mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine; - } else { - locals->ODMCombineEnablePerState[i][k] = true; - mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + + locals->ODMCombineEnablePerState[i][k] = false; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine; + if (mode_lib->vba.ODMCapability) { + if (locals->PlaneRequiredDISPCLKWithoutODMCombine > MaxMaxDispclkRoundedDown) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } else if (locals->DSCEnabled[k] && (locals->HActive[k] > DCN20_MAX_DSC_IMAGE_WIDTH)) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } else if (locals->HActive[k] > DCN20_MAX_420_IMAGE_WIDTH && locals->OutputFormat[k] == dm_420) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } } + if (locals->MinDPPCLKUsingSingleDPP[k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) <= mode_lib->vba.MaxDppclkRoundedDownToDFSGranularity && locals->SwathWidthYSingleDPP[k] <= locals->MaximumSwathWidth[k] && locals->ODMCombineEnablePerState[i][k] == dm_odm_combine_mode_disabled) { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c index af35b3bea909..e6617c958bb8 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c @@ -65,6 +65,7 @@ typedef struct { #define BPP_INVALID 0 #define BPP_BLENDED_PIPE 0xffffffff #define DCN21_MAX_DSC_IMAGE_WIDTH 5184 +#define DCN21_MAX_420_IMAGE_WIDTH 4096 static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib); static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation( @@ -3971,15 +3972,22 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l && i == mode_lib->vba.soc.num_states) mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine = mode_lib->vba.PixelClock[k] / 2 * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0); - if (mode_lib->vba.ODMCapability == false || - (locals->PlaneRequiredDISPCLKWithoutODMCombine <= MaxMaxDispclkRoundedDown - && (!locals->DSCEnabled[k] || locals->HActive[k] <= DCN21_MAX_DSC_IMAGE_WIDTH))) { - locals->ODMCombineEnablePerState[i][k] = false; - mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine; - } else { - locals->ODMCombineEnablePerState[i][k] = true; - mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + + locals->ODMCombineEnablePerState[i][k] = false; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine; + if (mode_lib->vba.ODMCapability) { + if (locals->PlaneRequiredDISPCLKWithoutODMCombine > MaxMaxDispclkRoundedDown) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } else if (locals->DSCEnabled[k] && (locals->HActive[k] > DCN21_MAX_DSC_IMAGE_WIDTH)) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } else if (locals->HActive[k] > DCN21_MAX_420_IMAGE_WIDTH && locals->OutputFormat[k] == dm_420) { + locals->ODMCombineEnablePerState[i][k] = true; + mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine; + } } + if (locals->MinDPPCLKUsingSingleDPP[k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) <= mode_lib->vba.MaxDppclkRoundedDownToDFSGranularity && locals->SwathWidthYSingleDPP[k] <= locals->MaximumSwathWidth[k] && locals->ODMCombineEnablePerState[i][k] == dm_odm_combine_mode_disabled) { diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h index e7a44df676ca..2875efd85467 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h @@ -763,8 +763,8 @@ struct vba_vars_st { double SwathWidthC[DC__NUM_DPP__MAX]; unsigned int BytePerPixelY[DC__NUM_DPP__MAX]; unsigned int BytePerPixelC[DC__NUM_DPP__MAX]; - long dummyinteger1; - long dummyinteger2; + unsigned int dummyinteger1; + unsigned int dummyinteger2; double FinalDRAMClockChangeLatency; double Tdmdl_vm[DC__NUM_DPP__MAX]; double Tdmdl[DC__NUM_DPP__MAX]; diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c index 8b78fcbfe746..87d682d25278 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c +++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c @@ -224,7 +224,8 @@ static void get_dsc_enc_caps( memset(dsc_enc_caps, 0, sizeof(struct dsc_enc_caps)); if (dsc) { - dsc->funcs->dsc_get_enc_caps(dsc_enc_caps, pixel_clock_100Hz); + if (!dsc->ctx->dc->debug.disable_dsc) + dsc->funcs->dsc_get_enc_caps(dsc_enc_caps, pixel_clock_100Hz); if (dsc->ctx->dc->debug.native422_support) dsc_enc_caps->color_formats.bits.YCBCR_NATIVE_422 = 1; } diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_srv.h index 8e23a7017588..f8917594036a 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_srv.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_srv.h @@ -231,6 +231,8 @@ struct dmub_srv_base_funcs { struct dmub_srv_hw_funcs { /* private: internal use only */ + void (*init)(struct dmub_srv *dmub); + void (*reset)(struct dmub_srv *dmub); void (*reset_release)(struct dmub_srv *dmub); @@ -417,6 +419,21 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, const struct dmub_srv_hw_params *params); /** + * dmub_srv_hw_reset() - puts the DMUB hardware in reset state if initialized + * @dmub: the dmub service + * + * Before destroying the DMUB service or releasing the backing framebuffer + * memory we'll need to put the DMCUB into reset first. + * + * A subsequent call to dmub_srv_hw_init() will re-enable the DMCUB. + * + * Return: + * DMUB_STATUS_OK - success + * DMUB_STATUS_INVALID - unspecified error + */ +enum dmub_status dmub_srv_hw_reset(struct dmub_srv *dmub); + +/** * dmub_srv_cmd_queue() - queues a command to the DMUB * @dmub: the dmub service * @cmd: the command to queue diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c index cd51c6138894..b2ca8e0dbac9 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c @@ -54,6 +54,19 @@ const struct dmub_srv_common_regs dmub_srv_dcn20_regs = { /* Shared functions. */ +static void dmub_dcn20_get_fb_base_offset(struct dmub_srv *dmub, + uint64_t *fb_base, + uint64_t *fb_offset) +{ + uint32_t tmp; + + REG_GET(DCN_VM_FB_LOCATION_BASE, FB_BASE, &tmp); + *fb_base = (uint64_t)tmp << 24; + + REG_GET(DCN_VM_FB_OFFSET, FB_OFFSET, &tmp); + *fb_offset = (uint64_t)tmp << 24; +} + static inline void dmub_dcn20_translate_addr(const union dmub_addr *addr_in, uint64_t fb_base, uint64_t fb_offset, @@ -67,6 +80,8 @@ void dmub_dcn20_reset(struct dmub_srv *dmub) REG_UPDATE(DMCUB_CNTL, DMCUB_SOFT_RESET, 1); REG_UPDATE(DMCUB_CNTL, DMCUB_ENABLE, 0); REG_UPDATE(MMHUBBUB_SOFT_RESET, DMUIF_SOFT_RESET, 1); + REG_WRITE(DMCUB_INBOX1_RPTR, 0); + REG_WRITE(DMCUB_INBOX1_WPTR, 0); } void dmub_dcn20_reset_release(struct dmub_srv *dmub) @@ -82,7 +97,9 @@ void dmub_dcn20_backdoor_load(struct dmub_srv *dmub, const struct dmub_window *cw1) { union dmub_addr offset; - uint64_t fb_base = dmub->fb_base, fb_offset = dmub->fb_offset; + uint64_t fb_base, fb_offset; + + dmub_dcn20_get_fb_base_offset(dmub, &fb_base, &fb_offset); REG_UPDATE(DMCUB_SEC_CNTL, DMCUB_SEC_RESET, 1); REG_UPDATE_2(DMCUB_MEM_CNTL, DMCUB_MEM_READ_SPACE, 0x3, @@ -118,7 +135,9 @@ void dmub_dcn20_setup_windows(struct dmub_srv *dmub, const struct dmub_window *cw6) { union dmub_addr offset; - uint64_t fb_base = dmub->fb_base, fb_offset = dmub->fb_offset; + uint64_t fb_base, fb_offset; + + dmub_dcn20_get_fb_base_offset(dmub, &fb_base, &fb_offset); dmub_dcn20_translate_addr(&cw2->offset, fb_base, fb_offset, &offset); @@ -173,8 +192,6 @@ void dmub_dcn20_setup_mailbox(struct dmub_srv *dmub, REG_WRITE(DMCUB_INBOX1_BASE_ADDRESS, 0x80000000); REG_WRITE(DMCUB_INBOX1_SIZE, inbox1->top - inbox1->base); - REG_WRITE(DMCUB_INBOX1_RPTR, 0); - REG_WRITE(DMCUB_INBOX1_WPTR, 0); } uint32_t dmub_dcn20_get_inbox1_rptr(struct dmub_srv *dmub) diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h index 53bfd4da69ad..04b0fa13153d 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h @@ -92,7 +92,9 @@ struct dmub_srv; DMUB_SR(DMCUB_SCRATCH14) \ DMUB_SR(DMCUB_SCRATCH15) \ DMUB_SR(CC_DC_PIPE_DIS) \ - DMUB_SR(MMHUBBUB_SOFT_RESET) + DMUB_SR(MMHUBBUB_SOFT_RESET) \ + DMUB_SR(DCN_VM_FB_LOCATION_BASE) \ + DMUB_SR(DCN_VM_FB_OFFSET) #define DMUB_COMMON_FIELDS() \ DMUB_SF(DMCUB_CNTL, DMCUB_ENABLE) \ @@ -121,7 +123,9 @@ struct dmub_srv; DMUB_SF(DMCUB_REGION4_TOP_ADDRESS, DMCUB_REGION4_TOP_ADDRESS) \ DMUB_SF(DMCUB_REGION4_TOP_ADDRESS, DMCUB_REGION4_ENABLE) \ DMUB_SF(CC_DC_PIPE_DIS, DC_DMCUB_ENABLE) \ - DMUB_SF(MMHUBBUB_SOFT_RESET, DMUIF_SOFT_RESET) + DMUB_SF(MMHUBBUB_SOFT_RESET, DMUIF_SOFT_RESET) \ + DMUB_SF(DCN_VM_FB_LOCATION_BASE, FB_BASE) \ + DMUB_SF(DCN_VM_FB_OFFSET, FB_OFFSET) struct dmub_srv_common_reg_offset { #define DMUB_SR(reg) uint32_t reg; diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c index dee676335d73..85a518bf8a76 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c @@ -312,6 +312,9 @@ enum dmub_status dmub_srv_is_hw_init(struct dmub_srv *dmub, bool *is_hw_init) if (!dmub->sw_init) return DMUB_STATUS_INVALID; + if (!dmub->hw_init) + return DMUB_STATUS_OK; + if (dmub->hw_funcs.is_hw_init) *is_hw_init = dmub->hw_funcs.is_hw_init(dmub); @@ -415,6 +418,22 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, return DMUB_STATUS_OK; } +enum dmub_status dmub_srv_hw_reset(struct dmub_srv *dmub) +{ + if (!dmub->sw_init) + return DMUB_STATUS_INVALID; + + if (dmub->hw_init == false) + return DMUB_STATUS_OK; + + if (dmub->hw_funcs.reset) + dmub->hw_funcs.reset(dmub); + + dmub->hw_init = false; + + return DMUB_STATUS_OK; +} + enum dmub_status dmub_srv_cmd_queue(struct dmub_srv *dmub, const struct dmub_cmd_header *cmd) { diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c index 1b278c42809a..cac09d500fda 100644 --- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c +++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c @@ -1673,129 +1673,6 @@ static bool map_regamma_hw_to_x_user( #define _EXTRA_POINTS 3 -bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf, - const struct dc_gamma *ramp, bool mapUserRamp, bool canRomBeUsed, - const struct freesync_hdr_tf_params *fs_params) -{ - struct dc_transfer_func_distributed_points *tf_pts = &output_tf->tf_pts; - struct dividers dividers; - - struct pwl_float_data *rgb_user = NULL; - struct pwl_float_data_ex *rgb_regamma = NULL; - struct gamma_pixel *axis_x = NULL; - struct pixel_gamma_point *coeff = NULL; - enum dc_transfer_func_predefined tf = TRANSFER_FUNCTION_SRGB; - bool ret = false; - - if (output_tf->type == TF_TYPE_BYPASS) - return false; - - /* we can use hardcoded curve for plain SRGB TF */ - if (output_tf->type == TF_TYPE_PREDEFINED && canRomBeUsed == true && - output_tf->tf == TRANSFER_FUNCTION_SRGB) { - if (ramp == NULL) - return true; - if ((ramp->is_identity && ramp->type != GAMMA_CS_TFM_1D) || - (!mapUserRamp && ramp->type == GAMMA_RGB_256)) - return true; - } - - output_tf->type = TF_TYPE_DISTRIBUTED_POINTS; - - if (ramp && ramp->type != GAMMA_CS_TFM_1D && - (mapUserRamp || ramp->type != GAMMA_RGB_256)) { - rgb_user = kvcalloc(ramp->num_entries + _EXTRA_POINTS, - sizeof(*rgb_user), - GFP_KERNEL); - if (!rgb_user) - goto rgb_user_alloc_fail; - - axis_x = kvcalloc(ramp->num_entries + 3, sizeof(*axis_x), - GFP_KERNEL); - if (!axis_x) - goto axis_x_alloc_fail; - - dividers.divider1 = dc_fixpt_from_fraction(3, 2); - dividers.divider2 = dc_fixpt_from_int(2); - dividers.divider3 = dc_fixpt_from_fraction(5, 2); - - build_evenly_distributed_points( - axis_x, - ramp->num_entries, - dividers); - - if (ramp->type == GAMMA_RGB_256 && mapUserRamp) - scale_gamma(rgb_user, ramp, dividers); - else if (ramp->type == GAMMA_RGB_FLOAT_1024) - scale_gamma_dx(rgb_user, ramp, dividers); - } - - rgb_regamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS, - sizeof(*rgb_regamma), - GFP_KERNEL); - if (!rgb_regamma) - goto rgb_regamma_alloc_fail; - - coeff = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS, sizeof(*coeff), - GFP_KERNEL); - if (!coeff) - goto coeff_alloc_fail; - - tf = output_tf->tf; - if (tf == TRANSFER_FUNCTION_PQ) { - tf_pts->end_exponent = 7; - tf_pts->x_point_at_y1_red = 125; - tf_pts->x_point_at_y1_green = 125; - tf_pts->x_point_at_y1_blue = 125; - - build_pq(rgb_regamma, - MAX_HW_POINTS, - coordinates_x, - output_tf->sdr_ref_white_level); - } else if (tf == TRANSFER_FUNCTION_GAMMA22 && - fs_params != NULL && fs_params->skip_tm == 0) { - build_freesync_hdr(rgb_regamma, - MAX_HW_POINTS, - coordinates_x, - fs_params); - } else if (tf == TRANSFER_FUNCTION_HLG) { - build_freesync_hdr(rgb_regamma, - MAX_HW_POINTS, - coordinates_x, - fs_params); - - } else { - tf_pts->end_exponent = 0; - tf_pts->x_point_at_y1_red = 1; - tf_pts->x_point_at_y1_green = 1; - tf_pts->x_point_at_y1_blue = 1; - - build_regamma(rgb_regamma, - MAX_HW_POINTS, - coordinates_x, tf); - } - map_regamma_hw_to_x_user(ramp, coeff, rgb_user, - coordinates_x, axis_x, rgb_regamma, - MAX_HW_POINTS, tf_pts, - (mapUserRamp || (ramp && ramp->type != GAMMA_RGB_256)) && - (ramp && ramp->type != GAMMA_CS_TFM_1D)); - - if (ramp && ramp->type == GAMMA_CS_TFM_1D) - apply_lut_1d(ramp, MAX_HW_POINTS, tf_pts); - - ret = true; - - kvfree(coeff); -coeff_alloc_fail: - kvfree(rgb_regamma); -rgb_regamma_alloc_fail: - kvfree(axis_x); -axis_x_alloc_fail: - kvfree(rgb_user); -rgb_user_alloc_fail: - return ret; -} - bool calculate_user_regamma_coeff(struct dc_transfer_func *output_tf, const struct regamma_lut *regamma) { @@ -2043,14 +1920,14 @@ rgb_user_alloc_fail: return ret; } - -bool mod_color_calculate_curve(enum dc_transfer_func_predefined trans, +static bool calculate_curve(enum dc_transfer_func_predefined trans, struct dc_transfer_func_distributed_points *points, + struct pwl_float_data_ex *rgb_regamma, + const struct freesync_hdr_tf_params *fs_params, uint32_t sdr_ref_white_level) { uint32_t i; bool ret = false; - struct pwl_float_data_ex *rgb_regamma = NULL; if (trans == TRANSFER_FUNCTION_UNITY || trans == TRANSFER_FUNCTION_LINEAR) { @@ -2060,68 +1937,33 @@ bool mod_color_calculate_curve(enum dc_transfer_func_predefined trans, points->x_point_at_y1_blue = 1; for (i = 0; i <= MAX_HW_POINTS ; i++) { - points->red[i] = coordinates_x[i].x; - points->green[i] = coordinates_x[i].x; - points->blue[i] = coordinates_x[i].x; + rgb_regamma[i].r = coordinates_x[i].x; + rgb_regamma[i].g = coordinates_x[i].x; + rgb_regamma[i].b = coordinates_x[i].x; } + ret = true; } else if (trans == TRANSFER_FUNCTION_PQ) { - rgb_regamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS, - sizeof(*rgb_regamma), - GFP_KERNEL); - if (!rgb_regamma) - goto rgb_regamma_alloc_fail; points->end_exponent = 7; points->x_point_at_y1_red = 125; points->x_point_at_y1_green = 125; points->x_point_at_y1_blue = 125; - build_pq(rgb_regamma, MAX_HW_POINTS, coordinates_x, sdr_ref_white_level); - for (i = 0; i <= MAX_HW_POINTS ; i++) { - points->red[i] = rgb_regamma[i].r; - points->green[i] = rgb_regamma[i].g; - points->blue[i] = rgb_regamma[i].b; - } - ret = true; - - kvfree(rgb_regamma); - } else if (trans == TRANSFER_FUNCTION_SRGB || - trans == TRANSFER_FUNCTION_BT709 || - trans == TRANSFER_FUNCTION_GAMMA22 || - trans == TRANSFER_FUNCTION_GAMMA24 || - trans == TRANSFER_FUNCTION_GAMMA26) { - rgb_regamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS, - sizeof(*rgb_regamma), - GFP_KERNEL); - if (!rgb_regamma) - goto rgb_regamma_alloc_fail; - points->end_exponent = 0; - points->x_point_at_y1_red = 1; - points->x_point_at_y1_green = 1; - points->x_point_at_y1_blue = 1; - build_regamma(rgb_regamma, + ret = true; + } else if (trans == TRANSFER_FUNCTION_GAMMA22 && + fs_params != NULL && fs_params->skip_tm == 0) { + build_freesync_hdr(rgb_regamma, MAX_HW_POINTS, coordinates_x, - trans); - for (i = 0; i <= MAX_HW_POINTS ; i++) { - points->red[i] = rgb_regamma[i].r; - points->green[i] = rgb_regamma[i].g; - points->blue[i] = rgb_regamma[i].b; - } - ret = true; + fs_params); - kvfree(rgb_regamma); + ret = true; } else if (trans == TRANSFER_FUNCTION_HLG) { - rgb_regamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS, - sizeof(*rgb_regamma), - GFP_KERNEL); - if (!rgb_regamma) - goto rgb_regamma_alloc_fail; points->end_exponent = 4; points->x_point_at_y1_red = 12; points->x_point_at_y1_green = 12; @@ -2131,18 +1973,127 @@ bool mod_color_calculate_curve(enum dc_transfer_func_predefined trans, MAX_HW_POINTS, coordinates_x, 80, 1000); - for (i = 0; i <= MAX_HW_POINTS ; i++) { - points->red[i] = rgb_regamma[i].r; - points->green[i] = rgb_regamma[i].g; - points->blue[i] = rgb_regamma[i].b; - } + + ret = true; + } else { + // trans == TRANSFER_FUNCTION_SRGB + // trans == TRANSFER_FUNCTION_BT709 + // trans == TRANSFER_FUNCTION_GAMMA22 + // trans == TRANSFER_FUNCTION_GAMMA24 + // trans == TRANSFER_FUNCTION_GAMMA26 + points->end_exponent = 0; + points->x_point_at_y1_red = 1; + points->x_point_at_y1_green = 1; + points->x_point_at_y1_blue = 1; + + build_regamma(rgb_regamma, + MAX_HW_POINTS, + coordinates_x, + trans); + ret = true; - kvfree(rgb_regamma); } -rgb_regamma_alloc_fail: + return ret; } +bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf, + const struct dc_gamma *ramp, bool mapUserRamp, bool canRomBeUsed, + const struct freesync_hdr_tf_params *fs_params) +{ + struct dc_transfer_func_distributed_points *tf_pts = &output_tf->tf_pts; + struct dividers dividers; + + struct pwl_float_data *rgb_user = NULL; + struct pwl_float_data_ex *rgb_regamma = NULL; + struct gamma_pixel *axis_x = NULL; + struct pixel_gamma_point *coeff = NULL; + enum dc_transfer_func_predefined tf = TRANSFER_FUNCTION_SRGB; + bool ret = false; + + if (output_tf->type == TF_TYPE_BYPASS) + return false; + + /* we can use hardcoded curve for plain SRGB TF */ + if (output_tf->type == TF_TYPE_PREDEFINED && canRomBeUsed == true && + output_tf->tf == TRANSFER_FUNCTION_SRGB) { + if (ramp == NULL) + return true; + if ((ramp->is_identity && ramp->type != GAMMA_CS_TFM_1D) || + (!mapUserRamp && ramp->type == GAMMA_RGB_256)) + return true; + } + + output_tf->type = TF_TYPE_DISTRIBUTED_POINTS; + + if (ramp && ramp->type != GAMMA_CS_TFM_1D && + (mapUserRamp || ramp->type != GAMMA_RGB_256)) { + rgb_user = kvcalloc(ramp->num_entries + _EXTRA_POINTS, + sizeof(*rgb_user), + GFP_KERNEL); + if (!rgb_user) + goto rgb_user_alloc_fail; + + axis_x = kvcalloc(ramp->num_entries + 3, sizeof(*axis_x), + GFP_KERNEL); + if (!axis_x) + goto axis_x_alloc_fail; + + dividers.divider1 = dc_fixpt_from_fraction(3, 2); + dividers.divider2 = dc_fixpt_from_int(2); + dividers.divider3 = dc_fixpt_from_fraction(5, 2); + + build_evenly_distributed_points( + axis_x, + ramp->num_entries, + dividers); + + if (ramp->type == GAMMA_RGB_256 && mapUserRamp) + scale_gamma(rgb_user, ramp, dividers); + else if (ramp->type == GAMMA_RGB_FLOAT_1024) + scale_gamma_dx(rgb_user, ramp, dividers); + } + + rgb_regamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS, + sizeof(*rgb_regamma), + GFP_KERNEL); + if (!rgb_regamma) + goto rgb_regamma_alloc_fail; + + coeff = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS, sizeof(*coeff), + GFP_KERNEL); + if (!coeff) + goto coeff_alloc_fail; + + tf = output_tf->tf; + + ret = calculate_curve(tf, + tf_pts, + rgb_regamma, + fs_params, + output_tf->sdr_ref_white_level); + + if (ret) { + map_regamma_hw_to_x_user(ramp, coeff, rgb_user, + coordinates_x, axis_x, rgb_regamma, + MAX_HW_POINTS, tf_pts, + (mapUserRamp || (ramp && ramp->type != GAMMA_RGB_256)) && + (ramp && ramp->type != GAMMA_CS_TFM_1D)); + + if (ramp && ramp->type == GAMMA_CS_TFM_1D) + apply_lut_1d(ramp, MAX_HW_POINTS, tf_pts); + } + + kvfree(coeff); +coeff_alloc_fail: + kvfree(rgb_regamma); +rgb_regamma_alloc_fail: + kvfree(axis_x); +axis_x_alloc_fail: + kvfree(rgb_user); +rgb_user_alloc_fail: + return ret; +} bool mod_color_calculate_degamma_curve(enum dc_transfer_func_predefined trans, struct dc_transfer_func_distributed_points *points) diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.h b/drivers/gpu/drm/amd/display/modules/color/color_gamma.h index 44ddea58523a..9994817a9a03 100644 --- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.h +++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.h @@ -103,10 +103,6 @@ bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf, bool mod_color_calculate_degamma_params(struct dc_transfer_func *output_tf, const struct dc_gamma *ramp, bool mapUserRamp); -bool mod_color_calculate_curve(enum dc_transfer_func_predefined trans, - struct dc_transfer_func_distributed_points *points, - uint32_t sdr_ref_white_level); - bool mod_color_calculate_degamma_curve(enum dc_transfer_func_predefined trans, struct dc_transfer_func_distributed_points *points); diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c index 6e5ecefe7d9d..b9992ebf77a6 100644 --- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c +++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c @@ -381,7 +381,7 @@ static void apply_fixed_refresh(struct core_freesync *core_freesync, bool update = false; unsigned int max_render_time_in_us = in_out_vrr->max_duration_in_us; - //Compute the exit refresh rate and exit frame duration + /* Compute the exit refresh rate and exit frame duration */ unsigned int exit_refresh_rate_in_milli_hz = ((1000000000/max_render_time_in_us) + (1000*FIXED_REFRESH_EXIT_MARGIN_IN_HZ)); unsigned int exit_frame_duration_in_us = 1000000000/exit_refresh_rate_in_milli_hz; diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h index f98d3d9ecb6d..af78e4f1be68 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h @@ -63,7 +63,7 @@ struct mod_hdcp_transition_input_hdcp1 { uint8_t hdcp_capable_dp; uint8_t binfo_read_dp; uint8_t r0p_available_dp; - uint8_t link_integiry_check; + uint8_t link_integrity_check; uint8_t reauth_request_check; uint8_t stream_encryption_dp; }; diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c index 04845e43df15..37670db64855 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c @@ -283,8 +283,8 @@ static enum mod_hdcp_status wait_for_ready(struct mod_hdcp *hdcp, hdcp, "bstatus_read")) goto out; if (!mod_hdcp_execute_and_set(check_link_integrity_dp, - &input->link_integiry_check, &status, - hdcp, "link_integiry_check")) + &input->link_integrity_check, &status, + hdcp, "link_integrity_check")) goto out; if (!mod_hdcp_execute_and_set(check_no_reauthentication_request_dp, &input->reauth_request_check, &status, @@ -431,8 +431,8 @@ static enum mod_hdcp_status authenticated_dp(struct mod_hdcp *hdcp, hdcp, "bstatus_read")) goto out; if (!mod_hdcp_execute_and_set(check_link_integrity_dp, - &input->link_integiry_check, &status, - hdcp, "link_integiry_check")) + &input->link_integrity_check, &status, + hdcp, "link_integrity_check")) goto out; if (!mod_hdcp_execute_and_set(check_no_reauthentication_request_dp, &input->reauth_request_check, &status, diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c index 21ebc62bb9d9..76edcbe51f71 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_transition.c @@ -241,7 +241,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_dp_transition(struct mod_hdcp *hdcp, } break; case D1_A4_AUTHENTICATED: - if (input->link_integiry_check != PASS || + if (input->link_integrity_check != PASS || input->reauth_request_check != PASS) { /* 1A-07: restart hdcp on a link integrity failure */ fail_and_restart_in_ms(0, &status, output); @@ -249,7 +249,7 @@ enum mod_hdcp_status mod_hdcp_hdcp1_dp_transition(struct mod_hdcp *hdcp, } break; case D1_A6_WAIT_FOR_READY: - if (input->link_integiry_check == FAIL || + if (input->link_integrity_check == FAIL || input->reauth_request_check == FAIL) { fail_and_restart_in_ms(0, &status, output); break; diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h index 87c84691b5be..bb2c9c7a18df 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h @@ -71,4 +71,7 @@ #define smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3 0x1d098UL #define smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3 0x1d09cUL +#define smnDF_CS_UMC_AON0_DramBaseAddress0 0x1c110UL +#define smnDF_CS_UMC_AON0_DramLimitAddress0 0x1c114UL + #endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h index 65e9f756e86e..7afa87c7ff54 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h @@ -53,4 +53,12 @@ #define DF_CS_UMC_AON0_DramBaseAddress0__IntLvAddrSel_MASK 0x00000E00L #define DF_CS_UMC_AON0_DramBaseAddress0__DramBaseAddr_MASK 0xFFFFF000L +//DF_CS_UMC_AON0_DramLimitAddress0 +#define DF_CS_UMC_AON0_DramLimitAddress0__DstFabricID__SHIFT 0x0 +#define DF_CS_UMC_AON0_DramLimitAddress0__AllowReqIO__SHIFT 0xa +#define DF_CS_UMC_AON0_DramLimitAddress0__DramLimitAddr__SHIFT 0xc +#define DF_CS_UMC_AON0_DramLimitAddress0__DstFabricID_MASK 0x000003FFL +#define DF_CS_UMC_AON0_DramLimitAddress0__AllowReqIO_MASK 0x00000400L +#define DF_CS_UMC_AON0_DramLimitAddress0__DramLimitAddr_MASK 0xFFFFF000L + #endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h index c9e3f6d849a8..ea316d8dcb37 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h @@ -2060,7 +2060,8 @@ // addressBlock: gc_sqdec //SQ_CONFIG -#define SQ_CONFIG__UNUSED__SHIFT 0x0 +#define SQ_CONFIG__DISABLE_BARRIER_WAITCNT__SHIFT 0x0 +#define SQ_CONFIG__UNUSED__SHIFT 0x1 #define SQ_CONFIG__OVERRIDE_ALU_BUSY__SHIFT 0x7 #define SQ_CONFIG__DEBUG_EN__SHIFT 0x8 #define SQ_CONFIG__DEBUG_SINGLE_MEMOP__SHIFT 0x9 @@ -2079,7 +2080,8 @@ #define SQ_CONFIG__DISABLE_SP_REDUNDANT_THREAD_GATING__SHIFT 0x1d #define SQ_CONFIG__DISABLE_FLAT_SOFT_CLAUSE__SHIFT 0x1e #define SQ_CONFIG__DISABLE_MIMG_SOFT_CLAUSE__SHIFT 0x1f -#define SQ_CONFIG__UNUSED_MASK 0x0000007FL +#define SQ_CONFIG__DISABLE_BARRIER_WAITCNT_MASK 0x00000001L +#define SQ_CONFIG__UNUSED_MASK 0x0000007EL #define SQ_CONFIG__OVERRIDE_ALU_BUSY_MASK 0x00000080L #define SQ_CONFIG__DEBUG_EN_MASK 0x00000100L #define SQ_CONFIG__DEBUG_SINGLE_MEMOP_MASK 0x00000200L diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_1_offset.h new file mode 100644 index 000000000000..f41556abfbbc --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_1_offset.h @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _gc_9_4_1_OFFSET_HEADER +#define _gc_9_4_1_OFFSET_HEADER + +// addressBlock: gc_grbmdec +// base address: 0x8000 +#define mmGRBM_CNTL 0x0000 +#define mmGRBM_CNTL_BASE_IDX 0 +#define mmGRBM_SKEW_CNTL 0x0001 +#define mmGRBM_SKEW_CNTL_BASE_IDX 0 +#define mmGRBM_STATUS2 0x0002 +#define mmGRBM_STATUS2_BASE_IDX 0 +#define mmGRBM_PWR_CNTL 0x0003 +#define mmGRBM_PWR_CNTL_BASE_IDX 0 +#define mmGRBM_STATUS 0x0004 +#define mmGRBM_STATUS_BASE_IDX 0 +#define mmGRBM_STATUS_SE0 0x0005 +#define mmGRBM_STATUS_SE0_BASE_IDX 0 +#define mmGRBM_STATUS_SE1 0x0006 +#define mmGRBM_STATUS_SE1_BASE_IDX 0 +#define mmGRBM_SOFT_RESET 0x0008 +#define mmGRBM_SOFT_RESET_BASE_IDX 0 +#define mmGRBM_GFX_CLKEN_CNTL 0x000c +#define mmGRBM_GFX_CLKEN_CNTL_BASE_IDX 0 +#define mmGRBM_WAIT_IDLE_CLOCKS 0x000d +#define mmGRBM_WAIT_IDLE_CLOCKS_BASE_IDX 0 +#define mmGRBM_STATUS_SE2 0x000e +#define mmGRBM_STATUS_SE2_BASE_IDX 0 +#define mmGRBM_STATUS_SE3 0x000f +#define mmGRBM_STATUS_SE3_BASE_IDX 0 +#define mmGRBM_READ_ERROR 0x0016 +#define mmGRBM_READ_ERROR_BASE_IDX 0 +#define mmGRBM_READ_ERROR2 0x0017 +#define mmGRBM_READ_ERROR2_BASE_IDX 0 +#define mmGRBM_INT_CNTL 0x0018 +#define mmGRBM_INT_CNTL_BASE_IDX 0 +#define mmGRBM_TRAP_OP 0x0019 +#define mmGRBM_TRAP_OP_BASE_IDX 0 +#define mmGRBM_TRAP_ADDR 0x001a +#define mmGRBM_TRAP_ADDR_BASE_IDX 0 +#define mmGRBM_TRAP_ADDR_MSK 0x001b +#define mmGRBM_TRAP_ADDR_MSK_BASE_IDX 0 +#define mmGRBM_TRAP_WD 0x001c +#define mmGRBM_TRAP_WD_BASE_IDX 0 +#define mmGRBM_TRAP_WD_MSK 0x001d +#define mmGRBM_TRAP_WD_MSK_BASE_IDX 0 +#define mmGRBM_DSM_BYPASS 0x001e +#define mmGRBM_DSM_BYPASS_BASE_IDX 0 +#define mmGRBM_WRITE_ERROR 0x001f +#define mmGRBM_WRITE_ERROR_BASE_IDX 0 +#define mmGRBM_IOV_ERROR 0x0020 +#define mmGRBM_IOV_ERROR_BASE_IDX 0 +#define mmGRBM_CHIP_REVISION 0x0021 +#define mmGRBM_CHIP_REVISION_BASE_IDX 0 +#define mmGRBM_GFX_CNTL 0x0022 +#define mmGRBM_GFX_CNTL_BASE_IDX 0 +#define mmGRBM_RSMU_CFG 0x0023 +#define mmGRBM_RSMU_CFG_BASE_IDX 0 +#define mmGRBM_IH_CREDIT 0x0024 +#define mmGRBM_IH_CREDIT_BASE_IDX 0 +#define mmGRBM_PWR_CNTL2 0x0025 +#define mmGRBM_PWR_CNTL2_BASE_IDX 0 +#define mmGRBM_UTCL2_INVAL_RANGE_START 0x0026 +#define mmGRBM_UTCL2_INVAL_RANGE_START_BASE_IDX 0 +#define mmGRBM_UTCL2_INVAL_RANGE_END 0x0027 +#define mmGRBM_UTCL2_INVAL_RANGE_END_BASE_IDX 0 +#define mmGRBM_RSMU_READ_ERROR 0x0028 +#define mmGRBM_RSMU_READ_ERROR_BASE_IDX 0 +#define mmGRBM_CHICKEN_BITS 0x0029 +#define mmGRBM_CHICKEN_BITS_BASE_IDX 0 +#define mmGRBM_FENCE_RANGE0 0x002a +#define mmGRBM_FENCE_RANGE0_BASE_IDX 0 +#define mmGRBM_FENCE_RANGE1 0x002b +#define mmGRBM_FENCE_RANGE1_BASE_IDX 0 +#define mmGRBM_NOWHERE 0x003f +#define mmGRBM_NOWHERE_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG0 0x0040 +#define mmGRBM_SCRATCH_REG0_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG1 0x0041 +#define mmGRBM_SCRATCH_REG1_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG2 0x0042 +#define mmGRBM_SCRATCH_REG2_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG3 0x0043 +#define mmGRBM_SCRATCH_REG3_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG4 0x0044 +#define mmGRBM_SCRATCH_REG4_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG5 0x0045 +#define mmGRBM_SCRATCH_REG5_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG6 0x0046 +#define mmGRBM_SCRATCH_REG6_BASE_IDX 0 +#define mmGRBM_SCRATCH_REG7 0x0047 +#define mmGRBM_SCRATCH_REG7_BASE_IDX 0 + +// addressBlock: gc_cppdec2 +// base address: 0xc600 +#define mmCPF_EDC_TAG_CNT 0x1189 +#define mmCPF_EDC_TAG_CNT_BASE_IDX 0 +#define mmCPF_EDC_ROQ_CNT 0x118a +#define mmCPF_EDC_ROQ_CNT_BASE_IDX 0 +#define mmCPG_EDC_TAG_CNT 0x118b +#define mmCPG_EDC_TAG_CNT_BASE_IDX 0 +#define mmCPG_EDC_DMA_CNT 0x118d +#define mmCPG_EDC_DMA_CNT_BASE_IDX 0 +#define mmCPC_EDC_SCRATCH_CNT 0x118e +#define mmCPC_EDC_SCRATCH_CNT_BASE_IDX 0 +#define mmCPC_EDC_UCODE_CNT 0x118f +#define mmCPC_EDC_UCODE_CNT_BASE_IDX 0 +#define mmDC_EDC_STATE_CNT 0x1191 +#define mmDC_EDC_STATE_CNT_BASE_IDX 0 +#define mmDC_EDC_CSINVOC_CNT 0x1192 +#define mmDC_EDC_CSINVOC_CNT_BASE_IDX 0 +#define mmDC_EDC_RESTORE_CNT 0x1193 +#define mmDC_EDC_RESTORE_CNT_BASE_IDX 0 + +// addressBlock: gc_gdsdec +// base address: 0x9700 +#define mmGDS_EDC_CNT 0x05c5 +#define mmGDS_EDC_CNT_BASE_IDX 0 +#define mmGDS_EDC_GRBM_CNT 0x05c6 +#define mmGDS_EDC_GRBM_CNT_BASE_IDX 0 +#define mmGDS_EDC_OA_DED 0x05c7 +#define mmGDS_EDC_OA_DED_BASE_IDX 0 +#define mmGDS_EDC_OA_PHY_CNT 0x05cb +#define mmGDS_EDC_OA_PHY_CNT_BASE_IDX 0 +#define mmGDS_EDC_OA_PIPE_CNT 0x05cc +#define mmGDS_EDC_OA_PIPE_CNT_BASE_IDX 0 + +// addressBlock: gc_shsdec +// base address: 0x9000 +#define mmSPI_EDC_CNT 0x0445 +#define mmSPI_EDC_CNT_BASE_IDX 0 + +// addressBlock: gc_sqdec +// base address: 0x8c00 +#define mmSQC_EDC_CNT2 0x032c +#define mmSQC_EDC_CNT2_BASE_IDX 0 +#define mmSQC_EDC_CNT3 0x032d +#define mmSQC_EDC_CNT3_BASE_IDX 0 +#define mmSQC_EDC_PARITY_CNT3 0x032e +#define mmSQC_EDC_PARITY_CNT3_BASE_IDX 0 +#define mmSQC_EDC_CNT 0x03a2 +#define mmSQC_EDC_CNT_BASE_IDX 0 +#define mmSQ_EDC_SEC_CNT 0x03a3 +#define mmSQ_EDC_SEC_CNT_BASE_IDX 0 +#define mmSQ_EDC_DED_CNT 0x03a4 +#define mmSQ_EDC_DED_CNT_BASE_IDX 0 +#define mmSQ_EDC_INFO 0x03a5 +#define mmSQ_EDC_INFO_BASE_IDX 0 +#define mmSQ_EDC_CNT 0x03a6 +#define mmSQ_EDC_CNT_BASE_IDX 0 + +// addressBlock: gc_tpdec +// base address: 0x9400 +#define mmTA_EDC_CNT 0x0586 +#define mmTA_EDC_CNT_BASE_IDX 0 + +// addressBlock: gc_tcdec +// base address: 0xac00 +#define mmTCP_EDC_CNT 0x0b17 +#define mmTCP_EDC_CNT_BASE_IDX 0 +#define mmTCP_EDC_CNT_NEW 0x0b18 +#define mmTCP_EDC_CNT_NEW_BASE_IDX 0 +#define mmTCP_ATC_EDC_GATCL1_CNT 0x12b1 +#define mmTCP_ATC_EDC_GATCL1_CNT_BASE_IDX 0 +#define mmTCI_EDC_CNT 0x0b60 +#define mmTCI_EDC_CNT_BASE_IDX 0 +#define mmTCC_EDC_CNT 0x0b82 +#define mmTCC_EDC_CNT_BASE_IDX 0 +#define mmTCC_EDC_CNT2 0x0b83 +#define mmTCC_EDC_CNT2_BASE_IDX 0 +#define mmTCA_EDC_CNT 0x0bc5 +#define mmTCA_EDC_CNT_BASE_IDX 0 + +// addressBlock: gc_tpdec +// base address: 0x9400 +#define mmTD_EDC_CNT 0x052e +#define mmTD_EDC_CNT_BASE_IDX 0 +#define mmTA_EDC_CNT 0x0586 +#define mmTA_EDC_CNT_BASE_IDX 0 + +// addressBlock: gc_ea_gceadec2 +// base address: 0x9c00 +#define mmGCEA_EDC_CNT 0x0706 +#define mmGCEA_EDC_CNT_BASE_IDX 0 +#define mmGCEA_EDC_CNT2 0x0707 +#define mmGCEA_EDC_CNT2_BASE_IDX 0 +#define mmGCEA_EDC_CNT3 0x071b +#define mmGCEA_EDC_CNT3_BASE_IDX 0 + +// addressBlock: gc_gfxudec +// base address: 0x30000 +#define mmSCRATCH_REG0 0x2040 +#define mmSCRATCH_REG0_BASE_IDX 1 +#define mmSCRATCH_REG1 0x2041 +#define mmSCRATCH_REG1_BASE_IDX 1 +#define mmSCRATCH_REG2 0x2042 +#define mmSCRATCH_REG2_BASE_IDX 1 +#define mmSCRATCH_REG3 0x2043 +#define mmSCRATCH_REG3_BASE_IDX 1 +#define mmSCRATCH_REG4 0x2044 +#define mmSCRATCH_REG4_BASE_IDX 1 +#define mmSCRATCH_REG5 0x2045 +#define mmSCRATCH_REG5_BASE_IDX 1 +#define mmSCRATCH_REG6 0x2046 +#define mmSCRATCH_REG6_BASE_IDX 1 +#define mmSCRATCH_REG7 0x2047 +#define mmSCRATCH_REG7_BASE_IDX 1 +#define mmGRBM_GFX_INDEX 0x2200 +#define mmGRBM_GFX_INDEX_BASE_IDX 1 + +// addressBlock: gc_utcl2_atcl2dec +// base address: 0xa000 +#define mmATC_L2_CACHE_4K_DSM_INDEX 0x080e +#define mmATC_L2_CACHE_4K_DSM_INDEX_BASE_IDX 0 +#define mmATC_L2_CACHE_2M_DSM_INDEX 0x080f +#define mmATC_L2_CACHE_2M_DSM_INDEX_BASE_IDX 0 +#define mmATC_L2_CACHE_4K_DSM_CNTL 0x0810 +#define mmATC_L2_CACHE_4K_DSM_CNTL_BASE_IDX 0 +#define mmATC_L2_CACHE_2M_DSM_CNTL 0x0811 +#define mmATC_L2_CACHE_2M_DSM_CNTL_BASE_IDX 0 + +// addressBlock: gc_utcl2_vml2pfdec +// base address: 0xa100 +#define mmVML2_MEM_ECC_INDEX 0x0860 +#define mmVML2_MEM_ECC_INDEX_BASE_IDX 0 +#define mmVML2_WALKER_MEM_ECC_INDEX 0x0861 +#define mmVML2_WALKER_MEM_ECC_INDEX_BASE_IDX 0 +#define mmUTCL2_MEM_ECC_INDEX 0x0862 +#define mmUTCL2_MEM_ECC_INDEX_BASE_IDX 0 + +#define mmVML2_MEM_ECC_CNTL 0x0863 +#define mmVML2_MEM_ECC_CNTL_BASE_IDX 0 +#define mmVML2_WALKER_MEM_ECC_CNTL 0x0864 +#define mmVML2_WALKER_MEM_ECC_CNTL_BASE_IDX 0 +#define mmUTCL2_MEM_ECC_CNTL 0x0865 +#define mmUTCL2_MEM_ECC_CNTL_BASE_IDX 0 + +// addressBlock: gc_rlcpdec +// base address: 0x3b000 +#define mmRLC_EDC_CNT 0x4d40 +#define mmRLC_EDC_CNT_BASE_IDX 1 +#define mmRLC_EDC_CNT2 0x4d41 +#define mmRLC_EDC_CNT2_BASE_IDX 1 + +#endif
\ No newline at end of file diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_1_sh_mask.h new file mode 100644 index 000000000000..f26246a600c6 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_4_1_sh_mask.h @@ -0,0 +1,748 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _gc_9_4_1_SH_MASK_HEADER +#define _gc_9_4_1_SH_MASK_HEADER + +// addressBlock: gc_cppdec2 +//CPF_EDC_TAG_CNT +#define CPF_EDC_TAG_CNT__DED_COUNT__SHIFT 0x0 +#define CPF_EDC_TAG_CNT__SEC_COUNT__SHIFT 0x2 +#define CPF_EDC_TAG_CNT__DED_COUNT_MASK 0x00000003L +#define CPF_EDC_TAG_CNT__SEC_COUNT_MASK 0x0000000CL +//CPF_EDC_ROQ_CNT +#define CPF_EDC_ROQ_CNT__DED_COUNT_ME1__SHIFT 0x0 +#define CPF_EDC_ROQ_CNT__SEC_COUNT_ME1__SHIFT 0x2 +#define CPF_EDC_ROQ_CNT__DED_COUNT_ME2__SHIFT 0x4 +#define CPF_EDC_ROQ_CNT__SEC_COUNT_ME2__SHIFT 0x6 +#define CPF_EDC_ROQ_CNT__DED_COUNT_ME1_MASK 0x00000003L +#define CPF_EDC_ROQ_CNT__SEC_COUNT_ME1_MASK 0x0000000CL +#define CPF_EDC_ROQ_CNT__DED_COUNT_ME2_MASK 0x00000030L +#define CPF_EDC_ROQ_CNT__SEC_COUNT_ME2_MASK 0x000000C0L +//CPG_EDC_TAG_CNT +#define CPG_EDC_TAG_CNT__DED_COUNT__SHIFT 0x0 +#define CPG_EDC_TAG_CNT__SEC_COUNT__SHIFT 0x2 +#define CPG_EDC_TAG_CNT__DED_COUNT_MASK 0x00000003L +#define CPG_EDC_TAG_CNT__SEC_COUNT_MASK 0x0000000CL +//CPG_EDC_DMA_CNT +#define CPG_EDC_DMA_CNT__ROQ_DED_COUNT__SHIFT 0x0 +#define CPG_EDC_DMA_CNT__ROQ_SEC_COUNT__SHIFT 0x2 +#define CPG_EDC_DMA_CNT__TAG_DED_COUNT__SHIFT 0x4 +#define CPG_EDC_DMA_CNT__TAG_SEC_COUNT__SHIFT 0x6 +#define CPG_EDC_DMA_CNT__ROQ_DED_COUNT_MASK 0x00000003L +#define CPG_EDC_DMA_CNT__ROQ_SEC_COUNT_MASK 0x0000000CL +#define CPG_EDC_DMA_CNT__TAG_DED_COUNT_MASK 0x00000030L +#define CPG_EDC_DMA_CNT__TAG_SEC_COUNT_MASK 0x000000C0L +//CPC_EDC_SCRATCH_CNT +#define CPC_EDC_SCRATCH_CNT__DED_COUNT__SHIFT 0x0 +#define CPC_EDC_SCRATCH_CNT__SEC_COUNT__SHIFT 0x2 +#define CPC_EDC_SCRATCH_CNT__DED_COUNT_MASK 0x00000003L +#define CPC_EDC_SCRATCH_CNT__SEC_COUNT_MASK 0x0000000CL +//CPC_EDC_UCODE_CNT +#define CPC_EDC_UCODE_CNT__DED_COUNT__SHIFT 0x0 +#define CPC_EDC_UCODE_CNT__SEC_COUNT__SHIFT 0x2 +#define CPC_EDC_UCODE_CNT__DED_COUNT_MASK 0x00000003L +#define CPC_EDC_UCODE_CNT__SEC_COUNT_MASK 0x0000000CL +//DC_EDC_STATE_CNT +#define DC_EDC_STATE_CNT__DED_COUNT_ME1__SHIFT 0x0 +#define DC_EDC_STATE_CNT__SEC_COUNT_ME1__SHIFT 0x2 +#define DC_EDC_STATE_CNT__DED_COUNT_ME1_MASK 0x00000003L +#define DC_EDC_STATE_CNT__SEC_COUNT_ME1_MASK 0x0000000CL +//DC_EDC_CSINVOC_CNT +#define DC_EDC_CSINVOC_CNT__DED_COUNT_ME1__SHIFT 0x0 +#define DC_EDC_CSINVOC_CNT__SEC_COUNT_ME1__SHIFT 0x2 +#define DC_EDC_CSINVOC_CNT__DED_COUNT1_ME1__SHIFT 0x4 +#define DC_EDC_CSINVOC_CNT__SEC_COUNT1_ME1__SHIFT 0x6 +#define DC_EDC_CSINVOC_CNT__DED_COUNT_ME1_MASK 0x00000003L +#define DC_EDC_CSINVOC_CNT__SEC_COUNT_ME1_MASK 0x0000000CL +#define DC_EDC_CSINVOC_CNT__DED_COUNT1_ME1_MASK 0x00000030L +#define DC_EDC_CSINVOC_CNT__SEC_COUNT1_ME1_MASK 0x000000C0L +//DC_EDC_RESTORE_CNT +#define DC_EDC_RESTORE_CNT__DED_COUNT_ME1__SHIFT 0x0 +#define DC_EDC_RESTORE_CNT__SEC_COUNT_ME1__SHIFT 0x2 +#define DC_EDC_RESTORE_CNT__DED_COUNT1_ME1__SHIFT 0x4 +#define DC_EDC_RESTORE_CNT__SEC_COUNT1_ME1__SHIFT 0x6 +#define DC_EDC_RESTORE_CNT__DED_COUNT_ME1_MASK 0x00000003L +#define DC_EDC_RESTORE_CNT__SEC_COUNT_ME1_MASK 0x0000000CL +#define DC_EDC_RESTORE_CNT__DED_COUNT1_ME1_MASK 0x00000030L +#define DC_EDC_RESTORE_CNT__SEC_COUNT1_ME1_MASK 0x000000C0L + +// addressBlock: gc_gdsdec +//GDS_EDC_CNT +#define GDS_EDC_CNT__GDS_MEM_DED__SHIFT 0x0 +#define GDS_EDC_CNT__GDS_MEM_SEC__SHIFT 0x4 +#define GDS_EDC_CNT__UNUSED__SHIFT 0x6 +#define GDS_EDC_CNT__GDS_MEM_DED_MASK 0x00000003L +#define GDS_EDC_CNT__GDS_MEM_SEC_MASK 0x00000030L +#define GDS_EDC_CNT__UNUSED_MASK 0xFFFFFFC0L +//GDS_EDC_GRBM_CNT +#define GDS_EDC_GRBM_CNT__DED__SHIFT 0x0 +#define GDS_EDC_GRBM_CNT__SEC__SHIFT 0x2 +#define GDS_EDC_GRBM_CNT__UNUSED__SHIFT 0x4 +#define GDS_EDC_GRBM_CNT__DED_MASK 0x00000003L +#define GDS_EDC_GRBM_CNT__SEC_MASK 0x0000000CL +#define GDS_EDC_GRBM_CNT__UNUSED_MASK 0xFFFFFFF0L +//GDS_EDC_OA_DED +#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED__SHIFT 0x0 +#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED__SHIFT 0x1 +#define GDS_EDC_OA_DED__ME0_CS_DED__SHIFT 0x2 +#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED__SHIFT 0x3 +#define GDS_EDC_OA_DED__ME1_PIPE0_DED__SHIFT 0x4 +#define GDS_EDC_OA_DED__ME1_PIPE1_DED__SHIFT 0x5 +#define GDS_EDC_OA_DED__ME1_PIPE2_DED__SHIFT 0x6 +#define GDS_EDC_OA_DED__ME1_PIPE3_DED__SHIFT 0x7 +#define GDS_EDC_OA_DED__ME2_PIPE0_DED__SHIFT 0x8 +#define GDS_EDC_OA_DED__ME2_PIPE1_DED__SHIFT 0x9 +#define GDS_EDC_OA_DED__ME2_PIPE2_DED__SHIFT 0xa +#define GDS_EDC_OA_DED__ME2_PIPE3_DED__SHIFT 0xb +#define GDS_EDC_OA_DED__UNUSED1__SHIFT 0xc +#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED_MASK 0x00000001L +#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED_MASK 0x00000002L +#define GDS_EDC_OA_DED__ME0_CS_DED_MASK 0x00000004L +#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED_MASK 0x00000008L +#define GDS_EDC_OA_DED__ME1_PIPE0_DED_MASK 0x00000010L +#define GDS_EDC_OA_DED__ME1_PIPE1_DED_MASK 0x00000020L +#define GDS_EDC_OA_DED__ME1_PIPE2_DED_MASK 0x00000040L +#define GDS_EDC_OA_DED__ME1_PIPE3_DED_MASK 0x00000080L +#define GDS_EDC_OA_DED__ME2_PIPE0_DED_MASK 0x00000100L +#define GDS_EDC_OA_DED__ME2_PIPE1_DED_MASK 0x00000200L +#define GDS_EDC_OA_DED__ME2_PIPE2_DED_MASK 0x00000400L +#define GDS_EDC_OA_DED__ME2_PIPE3_DED_MASK 0x00000800L +#define GDS_EDC_OA_DED__UNUSED1_MASK 0xFFFFF000L +//GDS_EDC_OA_PHY_CNT +#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC__SHIFT 0x0 +#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED__SHIFT 0x2 +#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC__SHIFT 0x4 +#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED__SHIFT 0x6 +#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SEC__SHIFT 0x8 +#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_DED__SHIFT 0xa +#define GDS_EDC_OA_PHY_CNT__UNUSED1__SHIFT 0xc +#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC_MASK 0x00000003L +#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED_MASK 0x0000000CL +#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC_MASK 0x00000030L +#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED_MASK 0x000000C0L +#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SEC_MASK 0x00000300L +#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_DED_MASK 0x00000C00L +#define GDS_EDC_OA_PHY_CNT__UNUSED1_MASK 0xFFFFF000L +//GDS_EDC_OA_PIPE_CNT +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC__SHIFT 0x0 +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED__SHIFT 0x2 +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC__SHIFT 0x4 +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED__SHIFT 0x6 +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC__SHIFT 0x8 +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED__SHIFT 0xa +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC__SHIFT 0xc +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED__SHIFT 0xe +#define GDS_EDC_OA_PIPE_CNT__UNUSED__SHIFT 0x10 +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC_MASK 0x00000003L +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED_MASK 0x0000000CL +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC_MASK 0x00000030L +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED_MASK 0x000000C0L +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC_MASK 0x00000300L +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED_MASK 0x00000C00L +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC_MASK 0x00003000L +#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED_MASK 0x0000C000L +#define GDS_EDC_OA_PIPE_CNT__UNUSED_MASK 0xFFFF0000L + +// addressBlock: gc_shsdec +//SPI_EDC_CNT +#define SPI_EDC_CNT__SPI_SR_MEM_SEC_COUNT__SHIFT 0x0 +#define SPI_EDC_CNT__SPI_SR_MEM_DED_COUNT__SHIFT 0x2 +#define SPI_EDC_CNT__SPI_GDS_EXPREQ_SEC_COUNT__SHIFT 0x4 +#define SPI_EDC_CNT__SPI_GDS_EXPREQ_DED_COUNT__SHIFT 0x6 +#define SPI_EDC_CNT__SPI_WB_GRANT_30_SEC_COUNT__SHIFT 0x8 +#define SPI_EDC_CNT__SPI_WB_GRANT_30_DED_COUNT__SHIFT 0xa +#define SPI_EDC_CNT__SPI_WB_GRANT_61_SEC_COUNT__SHIFT 0xc +#define SPI_EDC_CNT__SPI_WB_GRANT_61_DED_COUNT__SHIFT 0xe +#define SPI_EDC_CNT__SPI_LIFE_CNT_SEC_COUNT__SHIFT 0x10 +#define SPI_EDC_CNT__SPI_LIFE_CNT_DED_COUNT__SHIFT 0x12 +#define SPI_EDC_CNT__SPI_SR_MEM_SEC_COUNT_MASK 0x00000003L +#define SPI_EDC_CNT__SPI_SR_MEM_DED_COUNT_MASK 0x0000000CL +#define SPI_EDC_CNT__SPI_GDS_EXPREQ_SEC_COUNT_MASK 0x00000030L +#define SPI_EDC_CNT__SPI_GDS_EXPREQ_DED_COUNT_MASK 0x000000C0L +#define SPI_EDC_CNT__SPI_WB_GRANT_30_SEC_COUNT_MASK 0x00000300L +#define SPI_EDC_CNT__SPI_WB_GRANT_30_DED_COUNT_MASK 0x00000C00L +#define SPI_EDC_CNT__SPI_WB_GRANT_61_SEC_COUNT_MASK 0x00003000L +#define SPI_EDC_CNT__SPI_WB_GRANT_61_DED_COUNT_MASK 0x0000C000L +#define SPI_EDC_CNT__SPI_LIFE_CNT_SEC_COUNT_MASK 0x00030000L +#define SPI_EDC_CNT__SPI_LIFE_CNT_DED_COUNT_MASK 0x000C0000L + +// addressBlock: gc_sqdec +//SQC_EDC_CNT2 +#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_SEC_COUNT__SHIFT 0x0 +#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_DED_COUNT__SHIFT 0x2 +#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_SEC_COUNT__SHIFT 0x4 +#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_DED_COUNT__SHIFT 0x6 +#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_SEC_COUNT__SHIFT 0x8 +#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_DED_COUNT__SHIFT 0xa +#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_SEC_COUNT__SHIFT 0xc +#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_DED_COUNT__SHIFT 0xe +#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_SEC_COUNT__SHIFT 0x10 +#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_DED_COUNT__SHIFT 0x12 +#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_SEC_COUNT_MASK 0x00000003L +#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_DED_COUNT_MASK 0x0000000CL +#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_SEC_COUNT_MASK 0x00000030L +#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_DED_COUNT_MASK 0x000000C0L +#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_SEC_COUNT_MASK 0x00000300L +#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_DED_COUNT_MASK 0x00000C00L +#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_SEC_COUNT_MASK 0x00003000L +#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_DED_COUNT_MASK 0x0000C000L +#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_SEC_COUNT_MASK 0x00030000L +#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_DED_COUNT_MASK 0x000C0000L +//SQC_EDC_CNT3 +#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_SEC_COUNT__SHIFT 0x0 +#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_DED_COUNT__SHIFT 0x2 +#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_SEC_COUNT__SHIFT 0x4 +#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_DED_COUNT__SHIFT 0x6 +#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_SEC_COUNT__SHIFT 0x8 +#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_DED_COUNT__SHIFT 0xa +#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_SEC_COUNT__SHIFT 0xc +#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_DED_COUNT__SHIFT 0xe +#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_SEC_COUNT_MASK 0x00000003L +#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_DED_COUNT_MASK 0x0000000CL +#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_SEC_COUNT_MASK 0x00000030L +#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_DED_COUNT_MASK 0x000000C0L +#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_SEC_COUNT_MASK 0x00000300L +#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_DED_COUNT_MASK 0x00000C00L +#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_SEC_COUNT_MASK 0x00003000L +#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_DED_COUNT_MASK 0x0000C000L +//SQC_EDC_PARITY_CNT3 +#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_SEC_COUNT__SHIFT 0x0 +#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_DED_COUNT__SHIFT 0x2 +#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_SEC_COUNT__SHIFT 0x4 +#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_DED_COUNT__SHIFT 0x6 +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_SEC_COUNT__SHIFT 0x8 +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_DED_COUNT__SHIFT 0xa +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_SEC_COUNT__SHIFT 0xc +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_DED_COUNT__SHIFT 0xe +#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_SEC_COUNT__SHIFT 0x10 +#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_DED_COUNT__SHIFT 0x12 +#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_SEC_COUNT__SHIFT 0x14 +#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_DED_COUNT__SHIFT 0x16 +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_SEC_COUNT__SHIFT 0x18 +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_DED_COUNT__SHIFT 0x1a +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_SEC_COUNT__SHIFT 0x1c +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_DED_COUNT__SHIFT 0x1e +#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_SEC_COUNT_MASK 0x00000003L +#define SQC_EDC_PARITY_CNT3__INST_BANKA_UTCL1_MISS_FIFO_DED_COUNT_MASK 0x0000000CL +#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_SEC_COUNT_MASK 0x00000030L +#define SQC_EDC_PARITY_CNT3__INST_BANKA_MISS_FIFO_DED_COUNT_MASK 0x000000C0L +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_SEC_COUNT_MASK 0x00000300L +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_HIT_FIFO_DED_COUNT_MASK 0x00000C00L +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_SEC_COUNT_MASK 0x00003000L +#define SQC_EDC_PARITY_CNT3__DATA_BANKA_MISS_FIFO_DED_COUNT_MASK 0x0000C000L +#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_SEC_COUNT_MASK 0x00030000L +#define SQC_EDC_PARITY_CNT3__INST_BANKB_UTCL1_MISS_FIFO_DED_COUNT_MASK 0x000C0000L +#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_SEC_COUNT_MASK 0x00300000L +#define SQC_EDC_PARITY_CNT3__INST_BANKB_MISS_FIFO_DED_COUNT_MASK 0x00C00000L +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_SEC_COUNT_MASK 0x03000000L +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_HIT_FIFO_DED_COUNT_MASK 0x0C000000L +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_SEC_COUNT_MASK 0x30000000L +#define SQC_EDC_PARITY_CNT3__DATA_BANKB_MISS_FIFO_DED_COUNT_MASK 0xC0000000L +//SQC_EDC_CNT +#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_SEC_COUNT__SHIFT 0x0 +#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_DED_COUNT__SHIFT 0x2 +#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_SEC_COUNT__SHIFT 0x4 +#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_DED_COUNT__SHIFT 0x6 +#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_SEC_COUNT__SHIFT 0x8 +#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_DED_COUNT__SHIFT 0xa +#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_SEC_COUNT__SHIFT 0xc +#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_DED_COUNT__SHIFT 0xe +#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_SEC_COUNT__SHIFT 0x10 +#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_DED_COUNT__SHIFT 0x12 +#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_SEC_COUNT__SHIFT 0x14 +#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_DED_COUNT__SHIFT 0x16 +#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_SEC_COUNT__SHIFT 0x18 +#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_DED_COUNT__SHIFT 0x1a +#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_SEC_COUNT__SHIFT 0x1c +#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_DED_COUNT__SHIFT 0x1e +#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_SEC_COUNT_MASK 0x00000003L +#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_DED_COUNT_MASK 0x0000000CL +#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_SEC_COUNT_MASK 0x00000030L +#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_DED_COUNT_MASK 0x000000C0L +#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_SEC_COUNT_MASK 0x00000300L +#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_DED_COUNT_MASK 0x00000C00L +#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_SEC_COUNT_MASK 0x00003000L +#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_DED_COUNT_MASK 0x0000C000L +#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_SEC_COUNT_MASK 0x00030000L +#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_DED_COUNT_MASK 0x000C0000L +#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_SEC_COUNT_MASK 0x00300000L +#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_DED_COUNT_MASK 0x00C00000L +#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_SEC_COUNT_MASK 0x03000000L +#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_DED_COUNT_MASK 0x0C000000L +#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_SEC_COUNT_MASK 0x30000000L +#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_DED_COUNT_MASK 0xC0000000L +//SQ_EDC_SEC_CNT +#define SQ_EDC_SEC_CNT__LDS_SEC__SHIFT 0x0 +#define SQ_EDC_SEC_CNT__SGPR_SEC__SHIFT 0x8 +#define SQ_EDC_SEC_CNT__VGPR_SEC__SHIFT 0x10 +#define SQ_EDC_SEC_CNT__LDS_SEC_MASK 0x000000FFL +#define SQ_EDC_SEC_CNT__SGPR_SEC_MASK 0x0000FF00L +#define SQ_EDC_SEC_CNT__VGPR_SEC_MASK 0x00FF0000L +//SQ_EDC_DED_CNT +#define SQ_EDC_DED_CNT__LDS_DED__SHIFT 0x0 +#define SQ_EDC_DED_CNT__SGPR_DED__SHIFT 0x8 +#define SQ_EDC_DED_CNT__VGPR_DED__SHIFT 0x10 +#define SQ_EDC_DED_CNT__LDS_DED_MASK 0x000000FFL +#define SQ_EDC_DED_CNT__SGPR_DED_MASK 0x0000FF00L +#define SQ_EDC_DED_CNT__VGPR_DED_MASK 0x00FF0000L +//SQ_EDC_INFO +#define SQ_EDC_INFO__WAVE_ID__SHIFT 0x0 +#define SQ_EDC_INFO__SIMD_ID__SHIFT 0x4 +#define SQ_EDC_INFO__SOURCE__SHIFT 0x6 +#define SQ_EDC_INFO__VM_ID__SHIFT 0x9 +#define SQ_EDC_INFO__WAVE_ID_MASK 0x0000000FL +#define SQ_EDC_INFO__SIMD_ID_MASK 0x00000030L +#define SQ_EDC_INFO__SOURCE_MASK 0x000001C0L +#define SQ_EDC_INFO__VM_ID_MASK 0x00001E00L +//SQ_EDC_CNT +#define SQ_EDC_CNT__LDS_D_SEC_COUNT__SHIFT 0x0 +#define SQ_EDC_CNT__LDS_D_DED_COUNT__SHIFT 0x2 +#define SQ_EDC_CNT__LDS_I_SEC_COUNT__SHIFT 0x4 +#define SQ_EDC_CNT__LDS_I_DED_COUNT__SHIFT 0x6 +#define SQ_EDC_CNT__SGPR_SEC_COUNT__SHIFT 0x8 +#define SQ_EDC_CNT__SGPR_DED_COUNT__SHIFT 0xa +#define SQ_EDC_CNT__VGPR0_SEC_COUNT__SHIFT 0xc +#define SQ_EDC_CNT__VGPR0_DED_COUNT__SHIFT 0xe +#define SQ_EDC_CNT__VGPR1_SEC_COUNT__SHIFT 0x10 +#define SQ_EDC_CNT__VGPR1_DED_COUNT__SHIFT 0x12 +#define SQ_EDC_CNT__VGPR2_SEC_COUNT__SHIFT 0x14 +#define SQ_EDC_CNT__VGPR2_DED_COUNT__SHIFT 0x16 +#define SQ_EDC_CNT__VGPR3_SEC_COUNT__SHIFT 0x18 +#define SQ_EDC_CNT__VGPR3_DED_COUNT__SHIFT 0x1a +#define SQ_EDC_CNT__LDS_D_SEC_COUNT_MASK 0x00000003L +#define SQ_EDC_CNT__LDS_D_DED_COUNT_MASK 0x0000000CL +#define SQ_EDC_CNT__LDS_I_SEC_COUNT_MASK 0x00000030L +#define SQ_EDC_CNT__LDS_I_DED_COUNT_MASK 0x000000C0L +#define SQ_EDC_CNT__SGPR_SEC_COUNT_MASK 0x00000300L +#define SQ_EDC_CNT__SGPR_DED_COUNT_MASK 0x00000C00L +#define SQ_EDC_CNT__VGPR0_SEC_COUNT_MASK 0x00003000L +#define SQ_EDC_CNT__VGPR0_DED_COUNT_MASK 0x0000C000L +#define SQ_EDC_CNT__VGPR1_SEC_COUNT_MASK 0x00030000L +#define SQ_EDC_CNT__VGPR1_DED_COUNT_MASK 0x000C0000L +#define SQ_EDC_CNT__VGPR2_SEC_COUNT_MASK 0x00300000L +#define SQ_EDC_CNT__VGPR2_DED_COUNT_MASK 0x00C00000L +#define SQ_EDC_CNT__VGPR3_SEC_COUNT_MASK 0x03000000L +#define SQ_EDC_CNT__VGPR3_DED_COUNT_MASK 0x0C000000L + +// addressBlock: gc_tpdec +//TA_EDC_CNT +#define TA_EDC_CNT__TA_FS_DFIFO_SEC_COUNT__SHIFT 0x0 +#define TA_EDC_CNT__TA_FS_DFIFO_DED_COUNT__SHIFT 0x2 +#define TA_EDC_CNT__TA_FS_AFIFO_SEC_COUNT__SHIFT 0x4 +#define TA_EDC_CNT__TA_FS_AFIFO_DED_COUNT__SHIFT 0x6 +#define TA_EDC_CNT__TA_FL_LFIFO_SEC_COUNT__SHIFT 0x8 +#define TA_EDC_CNT__TA_FL_LFIFO_DED_COUNT__SHIFT 0xa +#define TA_EDC_CNT__TA_FX_LFIFO_SEC_COUNT__SHIFT 0xc +#define TA_EDC_CNT__TA_FX_LFIFO_DED_COUNT__SHIFT 0xe +#define TA_EDC_CNT__TA_FS_CFIFO_SEC_COUNT__SHIFT 0x10 +#define TA_EDC_CNT__TA_FS_CFIFO_DED_COUNT__SHIFT 0x12 +#define TA_EDC_CNT__TA_FS_DFIFO_SEC_COUNT_MASK 0x00000003L +#define TA_EDC_CNT__TA_FS_DFIFO_DED_COUNT_MASK 0x0000000CL +#define TA_EDC_CNT__TA_FS_AFIFO_SEC_COUNT_MASK 0x00000030L +#define TA_EDC_CNT__TA_FS_AFIFO_DED_COUNT_MASK 0x000000C0L +#define TA_EDC_CNT__TA_FL_LFIFO_SEC_COUNT_MASK 0x00000300L +#define TA_EDC_CNT__TA_FL_LFIFO_DED_COUNT_MASK 0x00000C00L +#define TA_EDC_CNT__TA_FX_LFIFO_SEC_COUNT_MASK 0x00003000L +#define TA_EDC_CNT__TA_FX_LFIFO_DED_COUNT_MASK 0x0000C000L +#define TA_EDC_CNT__TA_FS_CFIFO_SEC_COUNT_MASK 0x00030000L +#define TA_EDC_CNT__TA_FS_CFIFO_DED_COUNT_MASK 0x000C0000L + +// addressBlock: gc_tcdec +//TCP_EDC_CNT +#define TCP_EDC_CNT__SEC_COUNT__SHIFT 0x0 +#define TCP_EDC_CNT__LFIFO_SED_COUNT__SHIFT 0x8 +#define TCP_EDC_CNT__DED_COUNT__SHIFT 0x10 +#define TCP_EDC_CNT__SEC_COUNT_MASK 0x000000FFL +#define TCP_EDC_CNT__LFIFO_SED_COUNT_MASK 0x0000FF00L +#define TCP_EDC_CNT__DED_COUNT_MASK 0x00FF0000L +//TCP_EDC_CNT_NEW +#define TCP_EDC_CNT_NEW__CACHE_RAM_SEC_COUNT__SHIFT 0x0 +#define TCP_EDC_CNT_NEW__CACHE_RAM_DED_COUNT__SHIFT 0x2 +#define TCP_EDC_CNT_NEW__LFIFO_RAM_SEC_COUNT__SHIFT 0x4 +#define TCP_EDC_CNT_NEW__LFIFO_RAM_DED_COUNT__SHIFT 0x6 +#define TCP_EDC_CNT_NEW__CMD_FIFO_SEC_COUNT__SHIFT 0x8 +#define TCP_EDC_CNT_NEW__CMD_FIFO_DED_COUNT__SHIFT 0xa +#define TCP_EDC_CNT_NEW__VM_FIFO_SEC_COUNT__SHIFT 0xc +#define TCP_EDC_CNT_NEW__VM_FIFO_DED_COUNT__SHIFT 0xe +#define TCP_EDC_CNT_NEW__DB_RAM_SED_COUNT__SHIFT 0x10 +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO0_SEC_COUNT__SHIFT 0x12 +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO0_DED_COUNT__SHIFT 0x14 +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO1_SEC_COUNT__SHIFT 0x16 +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO1_DED_COUNT__SHIFT 0x18 +#define TCP_EDC_CNT_NEW__CACHE_RAM_SEC_COUNT_MASK 0x00000003L +#define TCP_EDC_CNT_NEW__CACHE_RAM_DED_COUNT_MASK 0x0000000CL +#define TCP_EDC_CNT_NEW__LFIFO_RAM_SEC_COUNT_MASK 0x00000030L +#define TCP_EDC_CNT_NEW__LFIFO_RAM_DED_COUNT_MASK 0x000000C0L +#define TCP_EDC_CNT_NEW__CMD_FIFO_SEC_COUNT_MASK 0x00000300L +#define TCP_EDC_CNT_NEW__CMD_FIFO_DED_COUNT_MASK 0x00000C00L +#define TCP_EDC_CNT_NEW__VM_FIFO_SEC_COUNT_MASK 0x00003000L +#define TCP_EDC_CNT_NEW__VM_FIFO_DED_COUNT_MASK 0x0000C000L +#define TCP_EDC_CNT_NEW__DB_RAM_SED_COUNT_MASK 0x00030000L +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO0_SEC_COUNT_MASK 0x000C0000L +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO0_DED_COUNT_MASK 0x00300000L +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO1_SEC_COUNT_MASK 0x00C00000L +#define TCP_EDC_CNT_NEW__UTCL1_LFIFO1_DED_COUNT_MASK 0x03000000L +//TCP_ATC_EDC_GATCL1_CNT +#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC__SHIFT 0x0 +#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC_MASK 0x000000FFL +//TCI_EDC_CNT +#define TCI_EDC_CNT__WRITE_RAM_SEC_COUNT__SHIFT 0x0 +#define TCI_EDC_CNT__WRITE_RAM_DED_COUNT__SHIFT 0x2 +#define TCI_EDC_CNT__WRITE_RAM_SEC_COUNT_MASK 0x00000003L +#define TCI_EDC_CNT__WRITE_RAM_DED_COUNT_MASK 0x0000000CL +//TCA_EDC_CNT +#define TCA_EDC_CNT__HOLE_FIFO_SEC_COUNT__SHIFT 0x0 +#define TCA_EDC_CNT__HOLE_FIFO_DED_COUNT__SHIFT 0x2 +#define TCA_EDC_CNT__REQ_FIFO_SEC_COUNT__SHIFT 0x4 +#define TCA_EDC_CNT__REQ_FIFO_DED_COUNT__SHIFT 0x6 +#define TCA_EDC_CNT__HOLE_FIFO_SEC_COUNT_MASK 0x00000003L +#define TCA_EDC_CNT__HOLE_FIFO_DED_COUNT_MASK 0x0000000CL +#define TCA_EDC_CNT__REQ_FIFO_SEC_COUNT_MASK 0x00000030L +#define TCA_EDC_CNT__REQ_FIFO_DED_COUNT_MASK 0x000000C0L +//TCC_EDC_CNT +#define TCC_EDC_CNT__CACHE_DATA_SEC_COUNT__SHIFT 0x0 +#define TCC_EDC_CNT__CACHE_DATA_DED_COUNT__SHIFT 0x2 +#define TCC_EDC_CNT__CACHE_DIRTY_SEC_COUNT__SHIFT 0x4 +#define TCC_EDC_CNT__CACHE_DIRTY_DED_COUNT__SHIFT 0x6 +#define TCC_EDC_CNT__HIGH_RATE_TAG_SEC_COUNT__SHIFT 0x8 +#define TCC_EDC_CNT__HIGH_RATE_TAG_DED_COUNT__SHIFT 0xa +#define TCC_EDC_CNT__LOW_RATE_TAG_SEC_COUNT__SHIFT 0xc +#define TCC_EDC_CNT__LOW_RATE_TAG_DED_COUNT__SHIFT 0xe +#define TCC_EDC_CNT__SRC_FIFO_SEC_COUNT__SHIFT 0x10 +#define TCC_EDC_CNT__SRC_FIFO_DED_COUNT__SHIFT 0x12 +#define TCC_EDC_CNT__LATENCY_FIFO_SEC_COUNT__SHIFT 0x14 +#define TCC_EDC_CNT__LATENCY_FIFO_DED_COUNT__SHIFT 0x16 +#define TCC_EDC_CNT__LATENCY_FIFO_NEXT_RAM_SEC_COUNT__SHIFT 0x18 +#define TCC_EDC_CNT__LATENCY_FIFO_NEXT_RAM_DED_COUNT__SHIFT 0x1a +#define TCC_EDC_CNT__CACHE_DATA_SEC_COUNT_MASK 0x00000003L +#define TCC_EDC_CNT__CACHE_DATA_DED_COUNT_MASK 0x0000000CL +#define TCC_EDC_CNT__CACHE_DIRTY_SEC_COUNT_MASK 0x00000030L +#define TCC_EDC_CNT__CACHE_DIRTY_DED_COUNT_MASK 0x000000C0L +#define TCC_EDC_CNT__HIGH_RATE_TAG_SEC_COUNT_MASK 0x00000300L +#define TCC_EDC_CNT__HIGH_RATE_TAG_DED_COUNT_MASK 0x00000C00L +#define TCC_EDC_CNT__LOW_RATE_TAG_SEC_COUNT_MASK 0x00003000L +#define TCC_EDC_CNT__LOW_RATE_TAG_DED_COUNT_MASK 0x0000C000L +#define TCC_EDC_CNT__SRC_FIFO_SEC_COUNT_MASK 0x00030000L +#define TCC_EDC_CNT__SRC_FIFO_DED_COUNT_MASK 0x000C0000L +#define TCC_EDC_CNT__LATENCY_FIFO_SEC_COUNT_MASK 0x00300000L +#define TCC_EDC_CNT__LATENCY_FIFO_DED_COUNT_MASK 0x00C00000L +#define TCC_EDC_CNT__LATENCY_FIFO_NEXT_RAM_SEC_COUNT_MASK 0x03000000L +#define TCC_EDC_CNT__LATENCY_FIFO_NEXT_RAM_DED_COUNT_MASK 0x0C000000L +//TCC_EDC_CNT2 +#define TCC_EDC_CNT2__CACHE_TAG_PROBE_FIFO_SEC_COUNT__SHIFT 0x0 +#define TCC_EDC_CNT2__CACHE_TAG_PROBE_FIFO_DED_COUNT__SHIFT 0x2 +#define TCC_EDC_CNT2__UC_ATOMIC_FIFO_SEC_COUNT__SHIFT 0x4 +#define TCC_EDC_CNT2__UC_ATOMIC_FIFO_DED_COUNT__SHIFT 0x6 +#define TCC_EDC_CNT2__WRITE_CACHE_READ_SEC_COUNT__SHIFT 0x8 +#define TCC_EDC_CNT2__WRITE_CACHE_READ_DED_COUNT__SHIFT 0xa +#define TCC_EDC_CNT2__RETURN_CONTROL_SEC_COUNT__SHIFT 0xc +#define TCC_EDC_CNT2__RETURN_CONTROL_DED_COUNT__SHIFT 0xe +#define TCC_EDC_CNT2__IN_USE_TRANSFER_SEC_COUNT__SHIFT 0x10 +#define TCC_EDC_CNT2__IN_USE_TRANSFER_DED_COUNT__SHIFT 0x12 +#define TCC_EDC_CNT2__IN_USE_DEC_SEC_COUNT__SHIFT 0x14 +#define TCC_EDC_CNT2__IN_USE_DEC_DED_COUNT__SHIFT 0x16 +#define TCC_EDC_CNT2__WRITE_RETURN_SEC_COUNT__SHIFT 0x18 +#define TCC_EDC_CNT2__WRITE_RETURN_DED_COUNT__SHIFT 0x1a +#define TCC_EDC_CNT2__RETURN_DATA_SEC_COUNT__SHIFT 0x1c +#define TCC_EDC_CNT2__RETURN_DATA_DED_COUNT__SHIFT 0x1e +#define TCC_EDC_CNT2__CACHE_TAG_PROBE_FIFO_SEC_COUNT_MASK 0x00000003L +#define TCC_EDC_CNT2__CACHE_TAG_PROBE_FIFO_DED_COUNT_MASK 0x0000000CL +#define TCC_EDC_CNT2__UC_ATOMIC_FIFO_SEC_COUNT_MASK 0x00000030L +#define TCC_EDC_CNT2__UC_ATOMIC_FIFO_DED_COUNT_MASK 0x000000C0L +#define TCC_EDC_CNT2__WRITE_CACHE_READ_SEC_COUNT_MASK 0x00000300L +#define TCC_EDC_CNT2__WRITE_CACHE_READ_DED_COUNT_MASK 0x00000C00L +#define TCC_EDC_CNT2__RETURN_CONTROL_SEC_COUNT_MASK 0x00003000L +#define TCC_EDC_CNT2__RETURN_CONTROL_DED_COUNT_MASK 0x0000C000L +#define TCC_EDC_CNT2__IN_USE_TRANSFER_SEC_COUNT_MASK 0x00030000L +#define TCC_EDC_CNT2__IN_USE_TRANSFER_DED_COUNT_MASK 0x000C0000L +#define TCC_EDC_CNT2__IN_USE_DEC_SEC_COUNT_MASK 0x00300000L +#define TCC_EDC_CNT2__IN_USE_DEC_DED_COUNT_MASK 0x00C00000L +#define TCC_EDC_CNT2__WRITE_RETURN_SEC_COUNT_MASK 0x03000000L +#define TCC_EDC_CNT2__WRITE_RETURN_DED_COUNT_MASK 0x0C000000L +#define TCC_EDC_CNT2__RETURN_DATA_SEC_COUNT_MASK 0x30000000L +#define TCC_EDC_CNT2__RETURN_DATA_DED_COUNT_MASK 0xC0000000L + +// addressBlock: gc_tpdec +//TD_EDC_CNT +#define TD_EDC_CNT__SS_FIFO_LO_SEC_COUNT__SHIFT 0x0 +#define TD_EDC_CNT__SS_FIFO_LO_DED_COUNT__SHIFT 0x2 +#define TD_EDC_CNT__SS_FIFO_HI_SEC_COUNT__SHIFT 0x4 +#define TD_EDC_CNT__SS_FIFO_HI_DED_COUNT__SHIFT 0x6 +#define TD_EDC_CNT__CS_FIFO_SEC_COUNT__SHIFT 0x8 +#define TD_EDC_CNT__CS_FIFO_DED_COUNT__SHIFT 0xa +#define TD_EDC_CNT__SS_FIFO_LO_SEC_COUNT_MASK 0x00000003L +#define TD_EDC_CNT__SS_FIFO_LO_DED_COUNT_MASK 0x0000000CL +#define TD_EDC_CNT__SS_FIFO_HI_SEC_COUNT_MASK 0x00000030L +#define TD_EDC_CNT__SS_FIFO_HI_DED_COUNT_MASK 0x000000C0L +#define TD_EDC_CNT__CS_FIFO_SEC_COUNT_MASK 0x00000300L +#define TD_EDC_CNT__CS_FIFO_DED_COUNT_MASK 0x00000C00L +//TA_EDC_CNT +#define TA_EDC_CNT__TA_FS_DFIFO_SEC_COUNT__SHIFT 0x0 +#define TA_EDC_CNT__TA_FS_DFIFO_DED_COUNT__SHIFT 0x2 +#define TA_EDC_CNT__TA_FS_AFIFO_SEC_COUNT__SHIFT 0x4 +#define TA_EDC_CNT__TA_FS_AFIFO_DED_COUNT__SHIFT 0x6 +#define TA_EDC_CNT__TA_FL_LFIFO_SEC_COUNT__SHIFT 0x8 +#define TA_EDC_CNT__TA_FL_LFIFO_DED_COUNT__SHIFT 0xa +#define TA_EDC_CNT__TA_FX_LFIFO_SEC_COUNT__SHIFT 0xc +#define TA_EDC_CNT__TA_FX_LFIFO_DED_COUNT__SHIFT 0xe +#define TA_EDC_CNT__TA_FS_CFIFO_SEC_COUNT__SHIFT 0x10 +#define TA_EDC_CNT__TA_FS_CFIFO_DED_COUNT__SHIFT 0x12 +#define TA_EDC_CNT__TA_FS_DFIFO_SEC_COUNT_MASK 0x00000003L +#define TA_EDC_CNT__TA_FS_DFIFO_DED_COUNT_MASK 0x0000000CL +#define TA_EDC_CNT__TA_FS_AFIFO_SEC_COUNT_MASK 0x00000030L +#define TA_EDC_CNT__TA_FS_AFIFO_DED_COUNT_MASK 0x000000C0L +#define TA_EDC_CNT__TA_FL_LFIFO_SEC_COUNT_MASK 0x00000300L +#define TA_EDC_CNT__TA_FL_LFIFO_DED_COUNT_MASK 0x00000C00L +#define TA_EDC_CNT__TA_FX_LFIFO_SEC_COUNT_MASK 0x00003000L +#define TA_EDC_CNT__TA_FX_LFIFO_DED_COUNT_MASK 0x0000C000L +#define TA_EDC_CNT__TA_FS_CFIFO_SEC_COUNT_MASK 0x00030000L +#define TA_EDC_CNT__TA_FS_CFIFO_DED_COUNT_MASK 0x000C0000L + +// addressBlock: gc_ea_gceadec2 +//GCEA_EDC_CNT +#define GCEA_EDC_CNT__DRAMRD_CMDMEM_SEC_COUNT__SHIFT 0x0 +#define GCEA_EDC_CNT__DRAMRD_CMDMEM_DED_COUNT__SHIFT 0x2 +#define GCEA_EDC_CNT__DRAMWR_CMDMEM_SEC_COUNT__SHIFT 0x4 +#define GCEA_EDC_CNT__DRAMWR_CMDMEM_DED_COUNT__SHIFT 0x6 +#define GCEA_EDC_CNT__DRAMWR_DATAMEM_SEC_COUNT__SHIFT 0x8 +#define GCEA_EDC_CNT__DRAMWR_DATAMEM_DED_COUNT__SHIFT 0xa +#define GCEA_EDC_CNT__RRET_TAGMEM_SEC_COUNT__SHIFT 0xc +#define GCEA_EDC_CNT__RRET_TAGMEM_DED_COUNT__SHIFT 0xe +#define GCEA_EDC_CNT__WRET_TAGMEM_SEC_COUNT__SHIFT 0x10 +#define GCEA_EDC_CNT__WRET_TAGMEM_DED_COUNT__SHIFT 0x12 +#define GCEA_EDC_CNT__DRAMRD_PAGEMEM_SED_COUNT__SHIFT 0x14 +#define GCEA_EDC_CNT__DRAMWR_PAGEMEM_SED_COUNT__SHIFT 0x16 +#define GCEA_EDC_CNT__IORD_CMDMEM_SED_COUNT__SHIFT 0x18 +#define GCEA_EDC_CNT__IOWR_CMDMEM_SED_COUNT__SHIFT 0x1a +#define GCEA_EDC_CNT__IOWR_DATAMEM_SED_COUNT__SHIFT 0x1c +#define GCEA_EDC_CNT__MAM_AFMEM_SEC_COUNT__SHIFT 0x1e +#define GCEA_EDC_CNT__DRAMRD_CMDMEM_SEC_COUNT_MASK 0x00000003L +#define GCEA_EDC_CNT__DRAMRD_CMDMEM_DED_COUNT_MASK 0x0000000CL +#define GCEA_EDC_CNT__DRAMWR_CMDMEM_SEC_COUNT_MASK 0x00000030L +#define GCEA_EDC_CNT__DRAMWR_CMDMEM_DED_COUNT_MASK 0x000000C0L +#define GCEA_EDC_CNT__DRAMWR_DATAMEM_SEC_COUNT_MASK 0x00000300L +#define GCEA_EDC_CNT__DRAMWR_DATAMEM_DED_COUNT_MASK 0x00000C00L +#define GCEA_EDC_CNT__RRET_TAGMEM_SEC_COUNT_MASK 0x00003000L +#define GCEA_EDC_CNT__RRET_TAGMEM_DED_COUNT_MASK 0x0000C000L +#define GCEA_EDC_CNT__WRET_TAGMEM_SEC_COUNT_MASK 0x00030000L +#define GCEA_EDC_CNT__WRET_TAGMEM_DED_COUNT_MASK 0x000C0000L +#define GCEA_EDC_CNT__DRAMRD_PAGEMEM_SED_COUNT_MASK 0x00300000L +#define GCEA_EDC_CNT__DRAMWR_PAGEMEM_SED_COUNT_MASK 0x00C00000L +#define GCEA_EDC_CNT__IORD_CMDMEM_SED_COUNT_MASK 0x03000000L +#define GCEA_EDC_CNT__IOWR_CMDMEM_SED_COUNT_MASK 0x0C000000L +#define GCEA_EDC_CNT__IOWR_DATAMEM_SED_COUNT_MASK 0x30000000L +#define GCEA_EDC_CNT__MAM_AFMEM_SEC_COUNT_MASK 0xC0000000L +//GCEA_EDC_CNT2 +#define GCEA_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT__SHIFT 0x0 +#define GCEA_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT__SHIFT 0x2 +#define GCEA_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT__SHIFT 0x4 +#define GCEA_EDC_CNT2__GMIWR_CMDMEM_DED_COUNT__SHIFT 0x6 +#define GCEA_EDC_CNT2__GMIWR_DATAMEM_SEC_COUNT__SHIFT 0x8 +#define GCEA_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa +#define GCEA_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc +#define GCEA_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define GCEA_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define GCEA_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define GCEA_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define GCEA_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define GCEA_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define GCEA_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define GCEA_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define GCEA_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e +#define GCEA_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L +#define GCEA_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL +#define GCEA_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L +#define GCEA_EDC_CNT2__GMIWR_CMDMEM_DED_COUNT_MASK 0x000000C0L +#define GCEA_EDC_CNT2__GMIWR_DATAMEM_SEC_COUNT_MASK 0x00000300L +#define GCEA_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L +#define GCEA_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L +#define GCEA_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define GCEA_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define GCEA_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define GCEA_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define GCEA_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define GCEA_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define GCEA_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define GCEA_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define GCEA_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L +//GCEA_EDC_CNT3 +#define GCEA_EDC_CNT3__DRAMRD_PAGEMEM_DED_COUNT__SHIFT 0x0 +#define GCEA_EDC_CNT3__DRAMWR_PAGEMEM_DED_COUNT__SHIFT 0x2 +#define GCEA_EDC_CNT3__IORD_CMDMEM_DED_COUNT__SHIFT 0x4 +#define GCEA_EDC_CNT3__IOWR_CMDMEM_DED_COUNT__SHIFT 0x6 +#define GCEA_EDC_CNT3__IOWR_DATAMEM_DED_COUNT__SHIFT 0x8 +#define GCEA_EDC_CNT3__GMIRD_PAGEMEM_DED_COUNT__SHIFT 0xa +#define GCEA_EDC_CNT3__GMIWR_PAGEMEM_DED_COUNT__SHIFT 0xc +#define GCEA_EDC_CNT3__MAM_AFMEM_DED_COUNT__SHIFT 0xe +#define GCEA_EDC_CNT3__MAM_A0MEM_SEC_COUNT__SHIFT 0x10 +#define GCEA_EDC_CNT3__MAM_A0MEM_DED_COUNT__SHIFT 0x12 +#define GCEA_EDC_CNT3__MAM_A1MEM_SEC_COUNT__SHIFT 0x14 +#define GCEA_EDC_CNT3__MAM_A1MEM_DED_COUNT__SHIFT 0x16 +#define GCEA_EDC_CNT3__MAM_A2MEM_SEC_COUNT__SHIFT 0x18 +#define GCEA_EDC_CNT3__MAM_A2MEM_DED_COUNT__SHIFT 0x1a +#define GCEA_EDC_CNT3__MAM_A3MEM_SEC_COUNT__SHIFT 0x1c +#define GCEA_EDC_CNT3__MAM_A3MEM_DED_COUNT__SHIFT 0x1e +#define GCEA_EDC_CNT3__DRAMRD_PAGEMEM_DED_COUNT_MASK 0x00000003L +#define GCEA_EDC_CNT3__DRAMWR_PAGEMEM_DED_COUNT_MASK 0x0000000CL +#define GCEA_EDC_CNT3__IORD_CMDMEM_DED_COUNT_MASK 0x00000030L +#define GCEA_EDC_CNT3__IOWR_CMDMEM_DED_COUNT_MASK 0x000000C0L +#define GCEA_EDC_CNT3__IOWR_DATAMEM_DED_COUNT_MASK 0x00000300L +#define GCEA_EDC_CNT3__GMIRD_PAGEMEM_DED_COUNT_MASK 0x00000C00L +#define GCEA_EDC_CNT3__GMIWR_PAGEMEM_DED_COUNT_MASK 0x00003000L +#define GCEA_EDC_CNT3__MAM_AFMEM_DED_COUNT_MASK 0x0000C000L +#define GCEA_EDC_CNT3__MAM_A0MEM_SEC_COUNT_MASK 0x00030000L +#define GCEA_EDC_CNT3__MAM_A0MEM_DED_COUNT_MASK 0x000C0000L +#define GCEA_EDC_CNT3__MAM_A1MEM_SEC_COUNT_MASK 0x00300000L +#define GCEA_EDC_CNT3__MAM_A1MEM_DED_COUNT_MASK 0x00C00000L +#define GCEA_EDC_CNT3__MAM_A2MEM_SEC_COUNT_MASK 0x03000000L +#define GCEA_EDC_CNT3__MAM_A2MEM_DED_COUNT_MASK 0x0C000000L +#define GCEA_EDC_CNT3__MAM_A3MEM_SEC_COUNT_MASK 0x30000000L +#define GCEA_EDC_CNT3__MAM_A3MEM_DED_COUNT_MASK 0xC0000000L + +// addressBlock: gc_gfxudec +//GRBM_GFX_INDEX +#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT 0x0 +#define GRBM_GFX_INDEX__SH_INDEX__SHIFT 0x8 +#define GRBM_GFX_INDEX__SE_INDEX__SHIFT 0x10 +#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES__SHIFT 0x1d +#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT 0x1e +#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT 0x1f +#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK 0x000000FFL +#define GRBM_GFX_INDEX__SH_INDEX_MASK 0x0000FF00L +#define GRBM_GFX_INDEX__SE_INDEX_MASK 0x00FF0000L +#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK 0x20000000L +#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK 0x40000000L +#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK 0x80000000L + +// addressBlock: gc_utcl2_atcl2dec +//ATC_L2_CNTL +//ATC_L2_CACHE_4K_DSM_INDEX +#define ATC_L2_CACHE_4K_DSM_INDEX__INDEX__SHIFT 0x0 +#define ATC_L2_CACHE_4K_DSM_INDEX__INDEX_MASK 0x000000FFL +//ATC_L2_CACHE_2M_DSM_INDEX +#define ATC_L2_CACHE_2M_DSM_INDEX__INDEX__SHIFT 0x0 +#define ATC_L2_CACHE_2M_DSM_INDEX__INDEX_MASK 0x000000FFL +//ATC_L2_CACHE_4K_DSM_CNTL +#define ATC_L2_CACHE_4K_DSM_CNTL__SEC_COUNT__SHIFT 0xd +#define ATC_L2_CACHE_4K_DSM_CNTL__DED_COUNT__SHIFT 0xf +#define ATC_L2_CACHE_4K_DSM_CNTL__SEC_COUNT_MASK 0x00006000L +#define ATC_L2_CACHE_4K_DSM_CNTL__DED_COUNT_MASK 0x00018000L +//ATC_L2_CACHE_2M_DSM_CNTL +#define ATC_L2_CACHE_2M_DSM_CNTL__SEC_COUNT__SHIFT 0xd +#define ATC_L2_CACHE_2M_DSM_CNTL__DED_COUNT__SHIFT 0xf +#define ATC_L2_CACHE_2M_DSM_CNTL__SEC_COUNT_MASK 0x00006000L +#define ATC_L2_CACHE_2M_DSM_CNTL__DED_COUNT_MASK 0x00018000L + +// addressBlock: gc_utcl2_vml2pfdec +//VML2_MEM_ECC_INDEX +#define VML2_MEM_ECC_INDEX__INDEX__SHIFT 0x0 +#define VML2_MEM_ECC_INDEX__INDEX_MASK 0x000000FFL +//VML2_WALKER_MEM_ECC_INDEX +#define VML2_WALKER_MEM_ECC_INDEX__INDEX__SHIFT 0x0 +#define VML2_WALKER_MEM_ECC_INDEX__INDEX_MASK 0x000000FFL +//UTCL2_MEM_ECC_INDEX +#define UTCL2_MEM_ECC_INDEX__INDEX__SHIFT 0x0 +#define UTCL2_MEM_ECC_INDEX__INDEX_MASK 0x000000FFL +//VML2_MEM_ECC_CNTL +#define VML2_MEM_ECC_CNTL__SEC_COUNT__SHIFT 0xc +#define VML2_MEM_ECC_CNTL__DED_COUNT__SHIFT 0xe +#define VML2_MEM_ECC_CNTL__SEC_COUNT_MASK 0x00003000L +#define VML2_MEM_ECC_CNTL__DED_COUNT_MASK 0x0000C000L +//VML2_WALKER_MEM_ECC_CNTL +#define VML2_WALKER_MEM_ECC_CNTL__SEC_COUNT__SHIFT 0xc +#define VML2_WALKER_MEM_ECC_CNTL__DED_COUNT__SHIFT 0xe +#define VML2_WALKER_MEM_ECC_CNTL__SEC_COUNT_MASK 0x00003000L +#define VML2_WALKER_MEM_ECC_CNTL__DED_COUNT_MASK 0x0000C000L +//UTCL2_MEM_ECC_CNTL +#define UTCL2_MEM_ECC_CNTL__SEC_COUNT__SHIFT 0xc +#define UTCL2_MEM_ECC_CNTL__DED_COUNT__SHIFT 0xe +#define UTCL2_MEM_ECC_CNTL__SEC_COUNT_MASK 0x00003000L +#define UTCL2_MEM_ECC_CNTL__DED_COUNT_MASK 0x0000C000L + +// addressBlock: gc_rlcpdec +//RLC_EDC_CNT +#define RLC_EDC_CNT__RLCG_INSTR_RAM_SEC_COUNT__SHIFT 0x0 +#define RLC_EDC_CNT__RLCG_INSTR_RAM_DED_COUNT__SHIFT 0x2 +#define RLC_EDC_CNT__RLCG_SCRATCH_RAM_SEC_COUNT__SHIFT 0x4 +#define RLC_EDC_CNT__RLCG_SCRATCH_RAM_DED_COUNT__SHIFT 0x6 +#define RLC_EDC_CNT__RLCV_INSTR_RAM_SEC_COUNT__SHIFT 0x8 +#define RLC_EDC_CNT__RLCV_INSTR_RAM_DED_COUNT__SHIFT 0xa +#define RLC_EDC_CNT__RLCV_SCRATCH_RAM_SEC_COUNT__SHIFT 0xc +#define RLC_EDC_CNT__RLCV_SCRATCH_RAM_DED_COUNT__SHIFT 0xe +#define RLC_EDC_CNT__RLC_TCTAG_RAM_SEC_COUNT__SHIFT 0x10 +#define RLC_EDC_CNT__RLC_TCTAG_RAM_DED_COUNT__SHIFT 0x12 +#define RLC_EDC_CNT__RLC_SPM_SCRATCH_RAM_SEC_COUNT__SHIFT 0x14 +#define RLC_EDC_CNT__RLC_SPM_SCRATCH_RAM_DED_COUNT__SHIFT 0x16 +#define RLC_EDC_CNT__RLC_SRM_DATA_RAM_SEC_COUNT__SHIFT 0x18 +#define RLC_EDC_CNT__RLC_SRM_DATA_RAM_DED_COUNT__SHIFT 0x1a +#define RLC_EDC_CNT__RLC_SRM_ADDR_RAM_SEC_COUNT__SHIFT 0x1c +#define RLC_EDC_CNT__RLC_SRM_ADDR_RAM_DED_COUNT__SHIFT 0x1e +#define RLC_EDC_CNT__RLCG_INSTR_RAM_SEC_COUNT_MASK 0x00000003L +#define RLC_EDC_CNT__RLCG_INSTR_RAM_DED_COUNT_MASK 0x0000000CL +#define RLC_EDC_CNT__RLCG_SCRATCH_RAM_SEC_COUNT_MASK 0x00000030L +#define RLC_EDC_CNT__RLCG_SCRATCH_RAM_DED_COUNT_MASK 0x000000C0L +#define RLC_EDC_CNT__RLCV_INSTR_RAM_SEC_COUNT_MASK 0x00000300L +#define RLC_EDC_CNT__RLCV_INSTR_RAM_DED_COUNT_MASK 0x00000C00L +#define RLC_EDC_CNT__RLCV_SCRATCH_RAM_SEC_COUNT_MASK 0x00003000L +#define RLC_EDC_CNT__RLCV_SCRATCH_RAM_DED_COUNT_MASK 0x0000C000L +#define RLC_EDC_CNT__RLC_TCTAG_RAM_SEC_COUNT_MASK 0x00030000L +#define RLC_EDC_CNT__RLC_TCTAG_RAM_DED_COUNT_MASK 0x000C0000L +#define RLC_EDC_CNT__RLC_SPM_SCRATCH_RAM_SEC_COUNT_MASK 0x00300000L +#define RLC_EDC_CNT__RLC_SPM_SCRATCH_RAM_DED_COUNT_MASK 0x00C00000L +#define RLC_EDC_CNT__RLC_SRM_DATA_RAM_SEC_COUNT_MASK 0x03000000L +#define RLC_EDC_CNT__RLC_SRM_DATA_RAM_DED_COUNT_MASK 0x0C000000L +#define RLC_EDC_CNT__RLC_SRM_ADDR_RAM_SEC_COUNT_MASK 0x30000000L +#define RLC_EDC_CNT__RLC_SRM_ADDR_RAM_DED_COUNT_MASK 0xC0000000L +//RLC_EDC_CNT2 +#define RLC_EDC_CNT2__RLC_SPM_SE0_SCRATCH_RAM_SEC_COUNT__SHIFT 0x0 +#define RLC_EDC_CNT2__RLC_SPM_SE0_SCRATCH_RAM_DED_COUNT__SHIFT 0x2 +#define RLC_EDC_CNT2__RLC_SPM_SE1_SCRATCH_RAM_SEC_COUNT__SHIFT 0x4 +#define RLC_EDC_CNT2__RLC_SPM_SE1_SCRATCH_RAM_DED_COUNT__SHIFT 0x6 +#define RLC_EDC_CNT2__RLC_SPM_SE2_SCRATCH_RAM_SEC_COUNT__SHIFT 0x8 +#define RLC_EDC_CNT2__RLC_SPM_SE2_SCRATCH_RAM_DED_COUNT__SHIFT 0xa +#define RLC_EDC_CNT2__RLC_SPM_SE3_SCRATCH_RAM_SEC_COUNT__SHIFT 0xc +#define RLC_EDC_CNT2__RLC_SPM_SE3_SCRATCH_RAM_DED_COUNT__SHIFT 0xe +#define RLC_EDC_CNT2__RLC_SPM_SE4_SCRATCH_RAM_SEC_COUNT__SHIFT 0x10 +#define RLC_EDC_CNT2__RLC_SPM_SE4_SCRATCH_RAM_DED_COUNT__SHIFT 0x12 +#define RLC_EDC_CNT2__RLC_SPM_SE5_SCRATCH_RAM_SEC_COUNT__SHIFT 0x14 +#define RLC_EDC_CNT2__RLC_SPM_SE5_SCRATCH_RAM_DED_COUNT__SHIFT 0x16 +#define RLC_EDC_CNT2__RLC_SPM_SE6_SCRATCH_RAM_SEC_COUNT__SHIFT 0x18 +#define RLC_EDC_CNT2__RLC_SPM_SE6_SCRATCH_RAM_DED_COUNT__SHIFT 0x1a +#define RLC_EDC_CNT2__RLC_SPM_SE7_SCRATCH_RAM_SEC_COUNT__SHIFT 0x1c +#define RLC_EDC_CNT2__RLC_SPM_SE7_SCRATCH_RAM_DED_COUNT__SHIFT 0x1e +#define RLC_EDC_CNT2__RLC_SPM_SE0_SCRATCH_RAM_SEC_COUNT_MASK 0x00000003L +#define RLC_EDC_CNT2__RLC_SPM_SE0_SCRATCH_RAM_DED_COUNT_MASK 0x0000000CL +#define RLC_EDC_CNT2__RLC_SPM_SE1_SCRATCH_RAM_SEC_COUNT_MASK 0x00000030L +#define RLC_EDC_CNT2__RLC_SPM_SE1_SCRATCH_RAM_DED_COUNT_MASK 0x000000C0L +#define RLC_EDC_CNT2__RLC_SPM_SE2_SCRATCH_RAM_SEC_COUNT_MASK 0x00000300L +#define RLC_EDC_CNT2__RLC_SPM_SE2_SCRATCH_RAM_DED_COUNT_MASK 0x00000C00L +#define RLC_EDC_CNT2__RLC_SPM_SE3_SCRATCH_RAM_SEC_COUNT_MASK 0x00003000L +#define RLC_EDC_CNT2__RLC_SPM_SE3_SCRATCH_RAM_DED_COUNT_MASK 0x0000C000L +#define RLC_EDC_CNT2__RLC_SPM_SE4_SCRATCH_RAM_SEC_COUNT_MASK 0x00030000L +#define RLC_EDC_CNT2__RLC_SPM_SE4_SCRATCH_RAM_DED_COUNT_MASK 0x000C0000L +#define RLC_EDC_CNT2__RLC_SPM_SE5_SCRATCH_RAM_SEC_COUNT_MASK 0x00300000L +#define RLC_EDC_CNT2__RLC_SPM_SE5_SCRATCH_RAM_DED_COUNT_MASK 0x00C00000L +#define RLC_EDC_CNT2__RLC_SPM_SE6_SCRATCH_RAM_SEC_COUNT_MASK 0x03000000L +#define RLC_EDC_CNT2__RLC_SPM_SE6_SCRATCH_RAM_DED_COUNT_MASK 0x0C000000L +#define RLC_EDC_CNT2__RLC_SPM_SE7_SCRATCH_RAM_SEC_COUNT_MASK 0x30000000L +#define RLC_EDC_CNT2__RLC_SPM_SE7_SCRATCH_RAM_DED_COUNT_MASK 0xC0000000L + +#endif
\ No newline at end of file diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h index 40dfbf16bd34..111a71b434e2 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h @@ -11185,6 +11185,14 @@ #define MMEA0_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA0_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA0_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA0_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA0_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA0_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA0_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA0_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA0_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA0_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA0_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA0_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA0_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA0_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -11193,6 +11201,14 @@ #define MMEA0_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA0_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA0_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA0_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA0_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA0_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA0_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA0_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA0_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA0_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA0_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA0_DSM_CNTL #define MMEA0_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA0_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 @@ -14197,6 +14213,14 @@ #define MMEA1_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA1_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA1_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA1_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA1_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA1_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA1_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA1_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA1_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA1_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA1_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA1_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA1_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA1_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -14205,6 +14229,14 @@ #define MMEA1_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA1_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA1_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA1_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA1_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA1_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA1_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA1_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA1_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA1_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA1_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA1_DSM_CNTL #define MMEA1_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA1_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 @@ -17209,6 +17241,14 @@ #define MMEA2_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA2_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA2_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA2_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA2_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA2_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA2_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA2_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA2_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA2_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA2_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA2_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA2_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA2_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -17217,6 +17257,14 @@ #define MMEA2_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA2_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA2_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA2_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA2_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA2_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA2_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA2_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA2_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA2_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA2_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA2_DSM_CNTL #define MMEA2_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA2_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 @@ -20221,6 +20269,14 @@ #define MMEA3_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA3_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA3_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA3_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA3_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA3_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA3_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA3_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA3_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA3_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA3_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA3_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA3_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA3_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -20229,6 +20285,14 @@ #define MMEA3_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA3_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA3_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA3_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA3_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA3_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA3_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA3_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA3_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA3_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA3_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA3_DSM_CNTL #define MMEA3_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA3_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 @@ -23233,6 +23297,14 @@ #define MMEA4_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA4_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA4_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA4_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA4_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA4_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA4_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA4_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA4_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA4_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA4_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA4_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA4_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA4_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -23241,6 +23313,14 @@ #define MMEA4_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA4_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA4_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA4_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA4_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA4_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA4_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA4_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA4_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA4_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA4_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA4_DSM_CNTL #define MMEA4_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA4_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 @@ -34952,6 +35032,14 @@ #define MMEA5_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA5_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA5_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA5_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA5_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA5_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA5_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA5_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA5_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA5_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA5_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA5_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA5_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA5_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -34960,6 +35048,14 @@ #define MMEA5_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA5_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA5_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA5_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA5_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA5_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA5_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA5_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA5_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA5_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA5_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA5_DSM_CNTL #define MMEA5_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA5_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 @@ -37964,6 +38060,14 @@ #define MMEA6_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA6_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA6_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA6_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA6_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA6_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA6_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA6_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA6_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA6_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA6_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA6_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA6_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA6_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -37972,6 +38076,14 @@ #define MMEA6_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA6_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA6_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA6_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA6_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA6_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA6_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA6_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA6_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA6_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA6_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA6_DSM_CNTL #define MMEA6_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA6_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 @@ -40976,6 +41088,14 @@ #define MMEA7_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT 0xa #define MMEA7_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT 0xc #define MMEA7_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT 0xe +#define MMEA7_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT 0x10 +#define MMEA7_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT 0x12 +#define MMEA7_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT 0x14 +#define MMEA7_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT 0x16 +#define MMEA7_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT 0x18 +#define MMEA7_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT 0x1a +#define MMEA7_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT 0x1c +#define MMEA7_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT 0x1e #define MMEA7_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK 0x00000003L #define MMEA7_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK 0x0000000CL #define MMEA7_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK 0x00000030L @@ -40984,6 +41104,14 @@ #define MMEA7_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK 0x00000C00L #define MMEA7_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK 0x00003000L #define MMEA7_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK 0x0000C000L +#define MMEA7_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK 0x00030000L +#define MMEA7_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK 0x000C0000L +#define MMEA7_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK 0x00300000L +#define MMEA7_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK 0x00C00000L +#define MMEA7_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK 0x03000000L +#define MMEA7_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK 0x0C000000L +#define MMEA7_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK 0x30000000L +#define MMEA7_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK 0xC0000000L //MMEA7_DSM_CNTL #define MMEA7_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT 0x0 #define MMEA7_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT 0x2 diff --git a/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c b/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c index 99469479e277..99ad4ddbe12f 100644 --- a/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c @@ -21,6 +21,7 @@ */ #include <linux/firmware.h> +#include <linux/pci.h> #include "pp_debug.h" #include "amdgpu.h" @@ -1137,6 +1138,23 @@ static int smu_smc_table_hw_init(struct smu_context *smu, ret = smu_system_features_control(smu, true); if (ret) return ret; + + if (adev->asic_type == CHIP_NAVI10) { + if ((adev->pdev->device == 0x731f && (adev->pdev->revision == 0xc2 || + adev->pdev->revision == 0xc3 || + adev->pdev->revision == 0xca || + adev->pdev->revision == 0xcb)) || + (adev->pdev->device == 0x66af && (adev->pdev->revision == 0xf3 || + adev->pdev->revision == 0xf4 || + adev->pdev->revision == 0xf5 || + adev->pdev->revision == 0xf6))) { + ret = smu_disable_umc_cdr_12gbps_workaround(smu); + if (ret) { + pr_err("Workaround failed to disable UMC CDR feature on 12Gbps SKU!\n"); + return ret; + } + } + } } if (adev->asic_type != CHIP_ARCTURUS) { ret = smu_notify_display_change(smu); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c index 4e8ab139bb3b..689072a312a7 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c @@ -1026,12 +1026,15 @@ static int smu10_get_clock_by_type_with_latency(struct pp_hwmgr *hwmgr, clocks->num_levels = 0; for (i = 0; i < pclk_vol_table->count; i++) { - clocks->data[i].clocks_in_khz = pclk_vol_table->entries[i].clk * 10; - clocks->data[i].latency_in_us = latency_required ? - smu10_get_mem_latency(hwmgr, - pclk_vol_table->entries[i].clk) : - 0; - clocks->num_levels++; + if (pclk_vol_table->entries[i].clk) { + clocks->data[clocks->num_levels].clocks_in_khz = + pclk_vol_table->entries[i].clk * 10; + clocks->data[clocks->num_levels].latency_in_us = latency_required ? + smu10_get_mem_latency(hwmgr, + pclk_vol_table->entries[i].clk) : + 0; + clocks->num_levels++; + } } return 0; @@ -1077,9 +1080,11 @@ static int smu10_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr, clocks->num_levels = 0; for (i = 0; i < pclk_vol_table->count; i++) { - clocks->data[i].clocks_in_khz = pclk_vol_table->entries[i].clk * 10; - clocks->data[i].voltage_in_mv = pclk_vol_table->entries[i].vol; - clocks->num_levels++; + if (pclk_vol_table->entries[i].clk) { + clocks->data[clocks->num_levels].clocks_in_khz = pclk_vol_table->entries[i].clk * 10; + clocks->data[clocks->num_levels].voltage_in_mv = pclk_vol_table->entries[i].vol; + clocks->num_levels++; + } } return 0; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c index d70abada66bf..bf04cfefb283 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c @@ -720,7 +720,7 @@ static int smu7_setup_dpm_tables_v0(struct pp_hwmgr *hwmgr) data->dpm_table.vddc_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].v; data->dpm_table.vddc_table.dpm_levels[i].param1 = std_voltage_table->entries[i].Leakage; /* param1 is for corresponding std voltage */ - data->dpm_table.vddc_table.dpm_levels[i].enabled = 1; + data->dpm_table.vddc_table.dpm_levels[i].enabled = true; } data->dpm_table.vddc_table.count = allowed_vdd_sclk_table->count; @@ -730,7 +730,7 @@ static int smu7_setup_dpm_tables_v0(struct pp_hwmgr *hwmgr) /* Initialize Vddci DPM table based on allow Mclk values */ for (i = 0; i < allowed_vdd_mclk_table->count; i++) { data->dpm_table.vddci_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].v; - data->dpm_table.vddci_table.dpm_levels[i].enabled = 1; + data->dpm_table.vddci_table.dpm_levels[i].enabled = true; } data->dpm_table.vddci_table.count = allowed_vdd_mclk_table->count; } @@ -744,7 +744,7 @@ static int smu7_setup_dpm_tables_v0(struct pp_hwmgr *hwmgr) */ for (i = 0; i < allowed_vdd_mclk_table->count; i++) { data->dpm_table.mvdd_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].v; - data->dpm_table.mvdd_table.dpm_levels[i].enabled = 1; + data->dpm_table.mvdd_table.dpm_levels[i].enabled = true; } data->dpm_table.mvdd_table.count = allowed_vdd_mclk_table->count; } diff --git a/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h index b0591a8dda41..97b6714e83e6 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h @@ -273,6 +273,7 @@ struct smu_table_context uint8_t thermal_controller_type; void *overdrive_table; + void *boot_overdrive_table; }; struct smu_dpm_context { @@ -565,6 +566,7 @@ struct pptable_funcs { int (*set_soft_freq_limited_range)(struct smu_context *smu, enum smu_clk_type clk_type, uint32_t min, uint32_t max); int (*override_pcie_parameters)(struct smu_context *smu); uint32_t (*get_pptable_power_limit)(struct smu_context *smu); + int (*disable_umc_cdr_12gbps_workaround)(struct smu_context *smu); }; int smu_load_microcode(struct smu_context *smu); diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_types.h b/drivers/gpu/drm/amd/powerplay/inc/smu_types.h index d8c9b7f91fcc..a5b4df146713 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_types.h +++ b/drivers/gpu/drm/amd/powerplay/inc/smu_types.h @@ -170,6 +170,8 @@ __SMU_DUMMY_MAP(SetSoftMinJpeg), \ __SMU_DUMMY_MAP(SetHardMinFclkByFreq), \ __SMU_DUMMY_MAP(DFCstateControl), \ + __SMU_DUMMY_MAP(DAL_DISABLE_DUMMY_PSTATE_CHANGE), \ + __SMU_DUMMY_MAP(DAL_ENABLE_DUMMY_PSTATE_CHANGE), \ #undef __SMU_DUMMY_MAP #define __SMU_DUMMY_MAP(type) SMU_MSG_##type diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h index 373861ddccd0..406bfd187ce8 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h +++ b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h @@ -120,7 +120,10 @@ #define PPSMC_MSG_GetVoltageByDpmOverdrive 0x45 #define PPSMC_MSG_BacoAudioD3PME 0x48 -#define PPSMC_Message_Count 0x49 +#define PPSMC_MSG_DALDisableDummyPstateChange 0x49 +#define PPSMC_MSG_DALEnableDummyPstateChange 0x4A + +#define PPSMC_Message_Count 0x4B typedef uint32_t PPSMC_Result; typedef uint32_t PPSMC_Msg; diff --git a/drivers/gpu/drm/amd/powerplay/navi10_ppt.c b/drivers/gpu/drm/amd/powerplay/navi10_ppt.c index 93c66c69ca28..19a9846b730e 100644 --- a/drivers/gpu/drm/amd/powerplay/navi10_ppt.c +++ b/drivers/gpu/drm/amd/powerplay/navi10_ppt.c @@ -119,6 +119,10 @@ static struct smu_11_0_cmn2aisc_mapping navi10_message_map[SMU_MSG_MAX_COUNT] = MSG_MAP(PowerDownJpeg, PPSMC_MSG_PowerDownJpeg), MSG_MAP(BacoAudioD3PME, PPSMC_MSG_BacoAudioD3PME), MSG_MAP(ArmD3, PPSMC_MSG_ArmD3), + MSG_MAP(DAL_DISABLE_DUMMY_PSTATE_CHANGE,PPSMC_MSG_DALDisableDummyPstateChange), + MSG_MAP(DAL_ENABLE_DUMMY_PSTATE_CHANGE, PPSMC_MSG_DALEnableDummyPstateChange), + MSG_MAP(GetVoltageByDpm, PPSMC_MSG_GetVoltageByDpm), + MSG_MAP(GetVoltageByDpmOverdrive, PPSMC_MSG_GetVoltageByDpmOverdrive), }; static struct smu_11_0_cmn2aisc_mapping navi10_clk_map[SMU_CLK_COUNT] = { @@ -737,6 +741,15 @@ static inline bool navi10_od_feature_is_supported(struct smu_11_0_overdrive_tabl return od_table->cap[feature]; } +static void navi10_od_setting_get_range(struct smu_11_0_overdrive_table *od_table, + enum SMU_11_0_ODSETTING_ID setting, + uint32_t *min, uint32_t *max) +{ + if (min) + *min = od_table->min[setting]; + if (max) + *max = od_table->max[setting]; +} static int navi10_print_clk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf) @@ -755,6 +768,7 @@ static int navi10_print_clk_levels(struct smu_context *smu, OverDriveTable_t *od_table = (OverDriveTable_t *)table_context->overdrive_table; struct smu_11_0_overdrive_table *od_settings = smu->od_settings; + uint32_t min_value, max_value; switch (clk_type) { case SMU_GFXCLK: @@ -843,7 +857,7 @@ static int navi10_print_clk_levels(struct smu_context *smu, if (!navi10_od_feature_is_supported(od_settings, SMU_11_0_ODFEATURE_UCLK_MAX)) break; size += sprintf(buf + size, "OD_MCLK:\n"); - size += sprintf(buf + size, "0: %uMHz\n", od_table->UclkFmax); + size += sprintf(buf + size, "1: %uMHz\n", od_table->UclkFmax); break; case SMU_OD_VDDC_CURVE: if (!smu->od_enabled || !od_table || !od_settings) @@ -868,6 +882,55 @@ static int navi10_print_clk_levels(struct smu_context *smu, size += sprintf(buf + size, "%d: %uMHz @ %umV\n", i, curve_settings[0], curve_settings[1] / NAVI10_VOLTAGE_SCALE); } break; + case SMU_OD_RANGE: + if (!smu->od_enabled || !od_table || !od_settings) + break; + size = sprintf(buf, "%s:\n", "OD_RANGE"); + + if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODFEATURE_GFXCLK_LIMITS)) { + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_GFXCLKFMIN, + &min_value, NULL); + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_GFXCLKFMAX, + NULL, &max_value); + size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + min_value, max_value); + } + + if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODFEATURE_UCLK_MAX)) { + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_UCLKFMAX, + &min_value, &max_value); + size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", + min_value, max_value); + } + + if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODFEATURE_GFXCLK_CURVE)) { + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P1, + &min_value, &max_value); + size += sprintf(buf + size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n", + min_value, max_value); + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P1, + &min_value, &max_value); + size += sprintf(buf + size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n", + min_value, max_value); + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P2, + &min_value, &max_value); + size += sprintf(buf + size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n", + min_value, max_value); + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P2, + &min_value, &max_value); + size += sprintf(buf + size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n", + min_value, max_value); + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P3, + &min_value, &max_value); + size += sprintf(buf + size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n", + min_value, max_value); + navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P3, + &min_value, &max_value); + size += sprintf(buf + size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n", + min_value, max_value); + } + + break; default: break; } @@ -949,6 +1012,8 @@ static int navi10_get_clock_by_type_with_latency(struct smu_context *smu, case SMU_GFXCLK: case SMU_DCEFCLK: case SMU_SOCCLK: + case SMU_MCLK: + case SMU_UCLK: ret = smu_get_dpm_level_count(smu, clk_type, &level_count); if (ret) return ret; @@ -1871,6 +1936,28 @@ static int navi10_od_setting_check_range(struct smu_11_0_overdrive_table *od_tab return 0; } +static int navi10_overdrive_get_gfx_clk_base_voltage(struct smu_context *smu, + uint16_t *voltage, + uint32_t freq) +{ + uint32_t param = (freq & 0xFFFF) | (PPCLK_GFXCLK << 16); + uint32_t value = 0; + int ret; + + ret = smu_send_smc_msg_with_param(smu, + SMU_MSG_GetVoltageByDpm, + param); + if (ret) { + pr_err("[GetBaseVoltage] failed to get GFXCLK AVFS voltage from SMU!"); + return ret; + } + + smu_read_smc_arg(smu, &value); + *voltage = (uint16_t)value; + + return 0; +} + static int navi10_setup_od_limits(struct smu_context *smu) { struct smu_11_0_overdrive_table *overdrive_table = NULL; struct smu_11_0_powerplay_table *powerplay_table = NULL; @@ -1890,23 +1977,54 @@ static int navi10_setup_od_limits(struct smu_context *smu) { } static int navi10_set_default_od_settings(struct smu_context *smu, bool initialize) { - OverDriveTable_t *od_table; + OverDriveTable_t *od_table, *boot_od_table; int ret = 0; ret = smu_v11_0_set_default_od_settings(smu, initialize, sizeof(OverDriveTable_t)); if (ret) return ret; + od_table = (OverDriveTable_t *)smu->smu_table.overdrive_table; + boot_od_table = (OverDriveTable_t *)smu->smu_table.boot_overdrive_table; if (initialize) { ret = navi10_setup_od_limits(smu); if (ret) { pr_err("Failed to retrieve board OD limits\n"); return ret; } + if (od_table) { + if (!od_table->GfxclkVolt1) { + ret = navi10_overdrive_get_gfx_clk_base_voltage(smu, + &od_table->GfxclkVolt1, + od_table->GfxclkFreq1); + if (ret) + od_table->GfxclkVolt1 = 0; + if (boot_od_table) + boot_od_table->GfxclkVolt1 = od_table->GfxclkVolt1; + } + if (!od_table->GfxclkVolt2) { + ret = navi10_overdrive_get_gfx_clk_base_voltage(smu, + &od_table->GfxclkVolt2, + od_table->GfxclkFreq2); + if (ret) + od_table->GfxclkVolt2 = 0; + if (boot_od_table) + boot_od_table->GfxclkVolt2 = od_table->GfxclkVolt2; + } + + if (!od_table->GfxclkVolt3) { + ret = navi10_overdrive_get_gfx_clk_base_voltage(smu, + &od_table->GfxclkVolt3, + od_table->GfxclkFreq3); + if (ret) + od_table->GfxclkVolt3 = 0; + if (boot_od_table) + boot_od_table->GfxclkVolt3 = od_table->GfxclkVolt3; + } + } } - od_table = (OverDriveTable_t *)smu->smu_table.overdrive_table; if (od_table) { navi10_dump_od_table(od_table); } @@ -2002,6 +2120,13 @@ static int navi10_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TABL return ret; od_table->UclkFmax = input[1]; break; + case PP_OD_RESTORE_DEFAULT_TABLE: + if (!(table_context->overdrive_table && table_context->boot_overdrive_table)) { + pr_err("Overdrive table was not initialized!\n"); + return -EINVAL; + } + memcpy(table_context->overdrive_table, table_context->boot_overdrive_table, sizeof(OverDriveTable_t)); + break; case PP_OD_COMMIT_DPM_TABLE: navi10_dump_od_table(od_table); ret = smu_update_table(smu, SMU_TABLE_OVERDRIVE, 0, (void *)od_table, true); @@ -2091,6 +2216,61 @@ static int navi10_run_btc(struct smu_context *smu) return ret; } +static int navi10_dummy_pstate_control(struct smu_context *smu, bool enable) +{ + int result = 0; + + if (!enable) + result = smu_send_smc_msg(smu, SMU_MSG_DAL_DISABLE_DUMMY_PSTATE_CHANGE); + else + result = smu_send_smc_msg(smu, SMU_MSG_DAL_ENABLE_DUMMY_PSTATE_CHANGE); + + return result; +} + +static int navi10_disable_umc_cdr_12gbps_workaround(struct smu_context *smu) +{ + uint32_t uclk_count, uclk_min, uclk_max; + uint32_t smu_version; + int ret = 0; + + ret = smu_get_smc_version(smu, NULL, &smu_version); + if (ret) + return ret; + + /* This workaround is available only for 42.50 or later SMC firmwares */ + if (smu_version < 0x2A3200) + return 0; + + ret = smu_get_dpm_level_count(smu, SMU_UCLK, &uclk_count); + if (ret) + return ret; + + ret = smu_get_dpm_freq_by_index(smu, SMU_UCLK, (uint16_t)0, &uclk_min); + if (ret) + return ret; + + ret = smu_get_dpm_freq_by_index(smu, SMU_UCLK, (uint16_t)(uclk_count - 1), &uclk_max); + if (ret) + return ret; + + /* Force UCLK out of the highest DPM */ + ret = smu_set_hard_freq_range(smu, SMU_UCLK, 0, uclk_min); + if (ret) + return ret; + + /* Revert the UCLK Hardmax */ + ret = smu_set_hard_freq_range(smu, SMU_UCLK, 0, uclk_max); + if (ret) + return ret; + + /* + * In this case, SMU already disabled dummy pstate during enablement + * of UCLK DPM, we have to re-enabled it. + * */ + return navi10_dummy_pstate_control(smu, true); +} + static const struct pptable_funcs navi10_ppt_funcs = { .tables_init = navi10_tables_init, .alloc_dpm_context = navi10_allocate_dpm_context, @@ -2185,6 +2365,7 @@ static const struct pptable_funcs navi10_ppt_funcs = { .od_edit_dpm_table = navi10_od_edit_dpm_table, .get_pptable_power_limit = navi10_get_pptable_power_limit, .run_btc = navi10_run_btc, + .disable_umc_cdr_12gbps_workaround = navi10_disable_umc_cdr_12gbps_workaround, }; void navi10_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/powerplay/smu_internal.h b/drivers/gpu/drm/amd/powerplay/smu_internal.h index 783319ec8bf9..7bd200ffcda8 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_internal.h +++ b/drivers/gpu/drm/amd/powerplay/smu_internal.h @@ -207,4 +207,7 @@ int smu_send_smc_msg(struct smu_context *smu, enum smu_message_type msg); #define smu_update_pcie_parameters(smu, pcie_gen_cap, pcie_width_cap) \ ((smu)->ppt_funcs->update_pcie_parameters ? (smu)->ppt_funcs->update_pcie_parameters((smu), (pcie_gen_cap), (pcie_width_cap)) : 0) +#define smu_disable_umc_cdr_12gbps_workaround(smu) \ + ((smu)->ppt_funcs->disable_umc_cdr_12gbps_workaround ? (smu)->ppt_funcs->disable_umc_cdr_12gbps_workaround((smu)) : 0) + #endif diff --git a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c index 02f8c9cb89d9..0dc49479a7eb 100644 --- a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c +++ b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c @@ -1882,6 +1882,12 @@ int smu_v11_0_set_default_od_settings(struct smu_context *smu, bool initialize, pr_err("Failed to export overdrive table!\n"); return ret; } + if (!table_context->boot_overdrive_table) { + table_context->boot_overdrive_table = kmemdup(table_context->overdrive_table, overdrive_table_size, GFP_KERNEL); + if (!table_context->boot_overdrive_table) { + return -ENOMEM; + } + } } ret = smu_update_table(smu, SMU_TABLE_OVERDRIVE, 0, table_context->overdrive_table, true); if (ret) { diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c index a3915bfcce81..275dbf65f1a0 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c @@ -128,20 +128,20 @@ int vega12_enable_smc_features(struct pp_hwmgr *hwmgr, if (enable) { PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_EnableSmuFeaturesLow, smu_features_low) == 0, - "[EnableDisableSMCFeatures] Attemp to enable SMU features Low failed!", + "[EnableDisableSMCFeatures] Attempt to enable SMU features Low failed!", return -EINVAL); PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_EnableSmuFeaturesHigh, smu_features_high) == 0, - "[EnableDisableSMCFeatures] Attemp to enable SMU features High failed!", + "[EnableDisableSMCFeatures] Attempt to enable SMU features High failed!", return -EINVAL); } else { PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_DisableSmuFeaturesLow, smu_features_low) == 0, - "[EnableDisableSMCFeatures] Attemp to disable SMU features Low failed!", + "[EnableDisableSMCFeatures] Attempt to disable SMU features Low failed!", return -EINVAL); PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_DisableSmuFeaturesHigh, smu_features_high) == 0, - "[EnableDisableSMCFeatures] Attemp to disable SMU features High failed!", + "[EnableDisableSMCFeatures] Attempt to disable SMU features High failed!", return -EINVAL); } @@ -158,13 +158,13 @@ int vega12_get_enabled_smc_features(struct pp_hwmgr *hwmgr, PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc(hwmgr, PPSMC_MSG_GetEnabledSmuFeaturesLow) == 0, - "[GetEnabledSMCFeatures] Attemp to get SMU features Low failed!", + "[GetEnabledSMCFeatures] Attempt to get SMU features Low failed!", return -EINVAL); smc_features_low = smu9_get_argument(hwmgr); PP_ASSERT_WITH_CODE(smu9_send_msg_to_smc(hwmgr, PPSMC_MSG_GetEnabledSmuFeaturesHigh) == 0, - "[GetEnabledSMCFeatures] Attemp to get SMU features High failed!", + "[GetEnabledSMCFeatures] Attempt to get SMU features High failed!", return -EINVAL); smc_features_high = smu9_get_argument(hwmgr); diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c index 0db57fb83d30..49e5ef3e3876 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c @@ -316,20 +316,20 @@ int vega20_enable_smc_features(struct pp_hwmgr *hwmgr, if (enable) { PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_EnableSmuFeaturesLow, smu_features_low)) == 0, - "[EnableDisableSMCFeatures] Attemp to enable SMU features Low failed!", + "[EnableDisableSMCFeatures] Attempt to enable SMU features Low failed!", return ret); PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_EnableSmuFeaturesHigh, smu_features_high)) == 0, - "[EnableDisableSMCFeatures] Attemp to enable SMU features High failed!", + "[EnableDisableSMCFeatures] Attempt to enable SMU features High failed!", return ret); } else { PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_DisableSmuFeaturesLow, smu_features_low)) == 0, - "[EnableDisableSMCFeatures] Attemp to disable SMU features Low failed!", + "[EnableDisableSMCFeatures] Attempt to disable SMU features Low failed!", return ret); PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_DisableSmuFeaturesHigh, smu_features_high)) == 0, - "[EnableDisableSMCFeatures] Attemp to disable SMU features High failed!", + "[EnableDisableSMCFeatures] Attempt to disable SMU features High failed!", return ret); } @@ -347,12 +347,12 @@ int vega20_get_enabled_smc_features(struct pp_hwmgr *hwmgr, PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc(hwmgr, PPSMC_MSG_GetEnabledSmuFeaturesLow)) == 0, - "[GetEnabledSMCFeatures] Attemp to get SMU features Low failed!", + "[GetEnabledSMCFeatures] Attempt to get SMU features Low failed!", return ret); smc_features_low = vega20_get_argument(hwmgr); PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc(hwmgr, PPSMC_MSG_GetEnabledSmuFeaturesHigh)) == 0, - "[GetEnabledSMCFeatures] Attemp to get SMU features High failed!", + "[GetEnabledSMCFeatures] Attempt to get SMU features High failed!", return ret); smc_features_high = vega20_get_argument(hwmgr); diff --git a/drivers/gpu/drm/amd/powerplay/vega20_ppt.c b/drivers/gpu/drm/amd/powerplay/vega20_ppt.c index 38febd5ca4da..4ad8d6c14ee5 100644 --- a/drivers/gpu/drm/amd/powerplay/vega20_ppt.c +++ b/drivers/gpu/drm/amd/powerplay/vega20_ppt.c @@ -1706,22 +1706,11 @@ static int vega20_set_default_od_settings(struct smu_context *smu, struct smu_table_context *table_context = &smu->smu_table; int ret; - if (initialize) { - if (table_context->overdrive_table) - return -EINVAL; - - table_context->overdrive_table = kzalloc(sizeof(OverDriveTable_t), GFP_KERNEL); - - if (!table_context->overdrive_table) - return -ENOMEM; - - ret = smu_update_table(smu, SMU_TABLE_OVERDRIVE, 0, - table_context->overdrive_table, false); - if (ret) { - pr_err("Failed to export over drive table!\n"); - return ret; - } + ret = smu_v11_0_set_default_od_settings(smu, initialize, sizeof(OverDriveTable_t)); + if (ret) + return ret; + if (initialize) { ret = vega20_set_default_od8_setttings(smu); if (ret) return ret; @@ -2778,12 +2767,11 @@ static int vega20_odn_edit_dpm_table(struct smu_context *smu, break; case PP_OD_RESTORE_DEFAULT_TABLE: - ret = smu_update_table(smu, SMU_TABLE_OVERDRIVE, 0, table_context->overdrive_table, false); - if (ret) { - pr_err("Failed to export over drive table!\n"); - return ret; + if (!(table_context->overdrive_table && table_context->boot_overdrive_table)) { + pr_err("Overdrive table was not initialized!\n"); + return -EINVAL; } - + memcpy(table_context->overdrive_table, table_context->boot_overdrive_table, sizeof(OverDriveTable_t)); break; case PP_OD_COMMIT_DPM_TABLE: diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c index 8f6100db90ed..1c894548dd72 100644 --- a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c +++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c @@ -751,9 +751,9 @@ static int pll_10nm_register(struct dsi_pll_10nm *pll_10nm) snprintf(parent4, 32, "dsi%d_pll_post_out_div_clk", pll_10nm->id); hw = clk_hw_register_mux(dev, clk_name, - (const char *[]){ + ((const char *[]){ parent, parent2, parent3, parent4 - }, 4, 0, pll_10nm->phy_cmn_mmio + + }), 4, 0, pll_10nm->phy_cmn_mmio + REG_DSI_10nm_PHY_CMN_CLK_CFG1, 0, 2, 0, NULL); if (IS_ERR(hw)) { diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_28nm.c b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_28nm.c index 8c99e01ae332..6dffd7f4a99b 100644 --- a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_28nm.c +++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_28nm.c @@ -554,9 +554,9 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm) snprintf(parent1, 32, "dsi%dvco_clk", pll_28nm->id); snprintf(parent2, 32, "dsi%dindirect_path_div2_clk", pll_28nm->id); clks[num++] = clk_register_mux(dev, clk_name, - (const char *[]){ + ((const char *[]){ parent1, parent2 - }, 2, CLK_SET_RATE_PARENT, pll_28nm->mmio + + }), 2, CLK_SET_RATE_PARENT, pll_28nm->mmio + REG_DSI_28nm_PHY_PLL_VREG_CFG, 1, 1, 0, NULL); snprintf(clk_name, 32, "dsi%dpllbyte", pll_28nm->id); diff --git a/drivers/gpu/drm/nouveau/dispnv50/core.h b/drivers/gpu/drm/nouveau/dispnv50/core.h index df8336b593f7..ff94f3f6f264 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/core.h +++ b/drivers/gpu/drm/nouveau/dispnv50/core.h @@ -6,6 +6,7 @@ struct nv50_core { const struct nv50_core_func *func; struct nv50_dmac chan; + bool assign_windows; }; int nv50_core_new(struct nouveau_drm *, struct nv50_core **); @@ -18,6 +19,10 @@ struct nv50_core_func { struct nvif_device *); void (*update)(struct nv50_core *, u32 *interlock, bool ntfy); + struct { + void (*owner)(struct nv50_core *); + } wndw; + const struct nv50_head_func *head; const struct nv50_outp_func { void (*ctrl)(struct nv50_core *, int or, u32 ctrl, @@ -48,6 +53,7 @@ int core917d_new(struct nouveau_drm *, s32, struct nv50_core **); int corec37d_new(struct nouveau_drm *, s32, struct nv50_core **); int corec37d_ntfy_wait_done(struct nouveau_bo *, u32, struct nvif_device *); void corec37d_update(struct nv50_core *, u32 *, bool); +void corec37d_wndw_owner(struct nv50_core *); extern const struct nv50_outp_func sorc37d; int corec57d_new(struct nouveau_drm *, s32, struct nv50_core **); diff --git a/drivers/gpu/drm/nouveau/dispnv50/corec37d.c b/drivers/gpu/drm/nouveau/dispnv50/corec37d.c index 40d9b654ab8c..3b36dc8d36b2 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/corec37d.c +++ b/drivers/gpu/drm/nouveau/dispnv50/corec37d.c @@ -25,6 +25,20 @@ #include <nouveau_bo.h> void +corec37d_wndw_owner(struct nv50_core *core) +{ + const u32 windows = 8; /*XXX*/ + u32 *push, i; + if ((push = evo_wait(&core->chan, 2 * windows))) { + for (i = 0; i < windows; i++) { + evo_mthd(push, 0x1000 + (i * 0x080), 1); + evo_data(push, i >> 1); + } + evo_kick(push, &core->chan); + } +} + +void corec37d_update(struct nv50_core *core, u32 *interlock, bool ntfy) { u32 *push; @@ -76,20 +90,18 @@ corec37d_init(struct nv50_core *core) { const u32 windows = 8; /*XXX*/ u32 *push, i; - if ((push = evo_wait(&core->chan, 2 + 6 * windows + 2))) { + if ((push = evo_wait(&core->chan, 2 + 5 * windows))) { evo_mthd(push, 0x0208, 1); evo_data(push, core->chan.sync.handle); for (i = 0; i < windows; i++) { - evo_mthd(push, 0x1000 + (i * 0x080), 3); - evo_data(push, i >> 1); + evo_mthd(push, 0x1004 + (i * 0x080), 2); evo_data(push, 0x0000001f); evo_data(push, 0x00000000); evo_mthd(push, 0x1010 + (i * 0x080), 1); evo_data(push, 0x00127fff); } - evo_mthd(push, 0x0200, 1); - evo_data(push, 0x00000001); evo_kick(push, &core->chan); + core->assign_windows = true; } } @@ -99,6 +111,7 @@ corec37d = { .ntfy_init = corec37d_ntfy_init, .ntfy_wait_done = corec37d_ntfy_wait_done, .update = corec37d_update, + .wndw.owner = corec37d_wndw_owner, .head = &headc37d, .sor = &sorc37d, }; diff --git a/drivers/gpu/drm/nouveau/dispnv50/corec57d.c b/drivers/gpu/drm/nouveau/dispnv50/corec57d.c index b606d68cda10..147adcd60937 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/corec57d.c +++ b/drivers/gpu/drm/nouveau/dispnv50/corec57d.c @@ -27,20 +27,18 @@ corec57d_init(struct nv50_core *core) { const u32 windows = 8; /*XXX*/ u32 *push, i; - if ((push = evo_wait(&core->chan, 2 + 6 * windows + 2))) { + if ((push = evo_wait(&core->chan, 2 + 5 * windows))) { evo_mthd(push, 0x0208, 1); evo_data(push, core->chan.sync.handle); for (i = 0; i < windows; i++) { - evo_mthd(push, 0x1000 + (i * 0x080), 3); - evo_data(push, i >> 1); + evo_mthd(push, 0x1004 + (i * 0x080), 2); evo_data(push, 0x0000000f); evo_data(push, 0x00000000); evo_mthd(push, 0x1010 + (i * 0x080), 1); evo_data(push, 0x00117fff); } - evo_mthd(push, 0x0200, 1); - evo_data(push, 0x00000001); evo_kick(push, &core->chan); + core->assign_windows = true; } } @@ -50,6 +48,7 @@ corec57d = { .ntfy_init = corec37d_ntfy_init, .ntfy_wait_done = corec37d_ntfy_wait_done, .update = corec37d_update, + .wndw.owner = corec37d_wndw_owner, .head = &headc57d, .sor = &sorc37d, }; diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 2f123082c85d..a3dc2ba19fb2 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -1933,6 +1933,7 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state) struct nouveau_drm *drm = nouveau_drm(dev); struct nv50_disp *disp = nv50_disp(dev); struct nv50_atom *atom = nv50_atom(state); + struct nv50_core *core = disp->core; struct nv50_outp_atom *outp, *outt; u32 interlock[NV50_DISP_INTERLOCK__SIZE] = {}; int i; @@ -2051,6 +2052,21 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state) } } + /* Update window->head assignment. + * + * This has to happen in an update that's not interlocked with + * any window channels to avoid hitting HW error checks. + * + *TODO: Proper handling of window ownership (Turing apparently + * supports non-fixed mappings). + */ + if (core->assign_windows) { + core->func->wndw.owner(core); + core->func->update(core, interlock, false); + core->assign_windows = false; + interlock[NV50_DISP_INTERLOCK_CORE] = 0; + } + /* Update plane(s). */ for_each_new_plane_in_state(state, plane, new_plane_state, i) { struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c index 3aa2cc3af1e2..c1032527f791 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c @@ -155,6 +155,12 @@ gv100_disp_intr_ctrl_disp(struct nv50_disp *disp) if (stat & 0x00000008) stat &= ~0x00000008; + if (stat & 0x00000080) { + u32 error = nvkm_mask(device, 0x611848, 0x00000000, 0x00000000); + nvkm_warn(subdev, "error %08x\n", error); + stat &= ~0x00000080; + } + if (stat & 0x00000100) { unsigned long wndws = nvkm_rd32(device, 0x611858); unsigned long other = nvkm_rd32(device, 0x61185c); diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c index 856526cb2caf..d07c7db0c815 100644 --- a/drivers/gpu/drm/radeon/radeon_display.c +++ b/drivers/gpu/drm/radeon/radeon_display.c @@ -127,6 +127,8 @@ static void dce5_crtc_load_lut(struct drm_crtc *crtc) DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id); + msleep(10); + WREG32(NI_INPUT_CSC_CONTROL + radeon_crtc->crtc_offset, (NI_INPUT_CSC_GRPH_MODE(NI_INPUT_CSC_BYPASS) | NI_INPUT_CSC_OVL_MODE(NI_INPUT_CSC_BYPASS))); @@ -672,7 +674,6 @@ static void radeon_crtc_init(struct drm_device *dev, int index) { struct radeon_device *rdev = dev->dev_private; struct radeon_crtc *radeon_crtc; - int i; radeon_crtc = kzalloc(sizeof(struct radeon_crtc) + (RADEONFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL); if (radeon_crtc == NULL) @@ -701,12 +702,6 @@ static void radeon_crtc_init(struct drm_device *dev, int index) radeon_crtc->mode_set.num_connectors = 0; #endif - for (i = 0; i < 256; i++) { - radeon_crtc->lut_r[i] = i << 2; - radeon_crtc->lut_g[i] = i << 2; - radeon_crtc->lut_b[i] = i << 2; - } - if (rdev->is_atom_bios && (ASIC_IS_AVIVO(rdev) || radeon_r4xx_atom)) radeon_atombios_init_crtc(dev, radeon_crtc); else diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h index fd470d6bf3f4..96565171d13e 100644 --- a/drivers/gpu/drm/radeon/radeon_mode.h +++ b/drivers/gpu/drm/radeon/radeon_mode.h @@ -327,7 +327,6 @@ enum radeon_flip_status { struct radeon_crtc { struct drm_crtc base; int crtc_id; - u16 lut_r[256], lut_g[256], lut_b[256]; bool enabled; bool can_tile; bool cursor_out_of_bounds; diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index ec79e8e5ad3c..63bccd201b97 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -45,7 +45,7 @@ * @guilty: atomic_t set to 1 when a job on this queue * is found to be guilty causing a timeout * - * Note: the sched_list should have atleast one element to schedule + * Note: the sched_list should have at least one element to schedule * the entity * * Returns 0 on success or a negative error code on failure. diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c index aa9e49f04988..bd268028fb3d 100644 --- a/drivers/gpu/drm/tegra/drm.c +++ b/drivers/gpu/drm/tegra/drm.c @@ -1037,23 +1037,9 @@ void tegra_drm_free(struct tegra_drm *tegra, size_t size, void *virt, free_pages((unsigned long)virt, get_order(size)); } -static int host1x_drm_probe(struct host1x_device *dev) +static bool host1x_drm_wants_iommu(struct host1x_device *dev) { - struct drm_driver *driver = &tegra_drm_driver; struct iommu_domain *domain; - struct tegra_drm *tegra; - struct drm_device *drm; - int err; - - drm = drm_dev_alloc(driver, &dev->dev); - if (IS_ERR(drm)) - return PTR_ERR(drm); - - tegra = kzalloc(sizeof(*tegra), GFP_KERNEL); - if (!tegra) { - err = -ENOMEM; - goto put; - } /* * If the Tegra DRM clients are backed by an IOMMU, push buffers are @@ -1082,9 +1068,38 @@ static int host1x_drm_probe(struct host1x_device *dev) * up the device tree appropriately. This is considered an problem * of integration, so care must be taken for the DT to be consistent. */ - domain = iommu_get_domain_for_dev(drm->dev->parent); + domain = iommu_get_domain_for_dev(dev->dev.parent); + + /* + * Tegra20 and Tegra30 don't support addressing memory beyond the + * 32-bit boundary, so the regular GATHER opcodes will always be + * sufficient and whether or not the host1x is attached to an IOMMU + * doesn't matter. + */ + if (!domain && dma_get_mask(dev->dev.parent) <= DMA_BIT_MASK(32)) + return true; + + return domain != NULL; +} + +static int host1x_drm_probe(struct host1x_device *dev) +{ + struct drm_driver *driver = &tegra_drm_driver; + struct tegra_drm *tegra; + struct drm_device *drm; + int err; + + drm = drm_dev_alloc(driver, &dev->dev); + if (IS_ERR(drm)) + return PTR_ERR(drm); + + tegra = kzalloc(sizeof(*tegra), GFP_KERNEL); + if (!tegra) { + err = -ENOMEM; + goto put; + } - if (domain && iommu_present(&platform_bus_type)) { + if (host1x_drm_wants_iommu(dev) && iommu_present(&platform_bus_type)) { tegra->domain = iommu_domain_alloc(&platform_bus_type); if (!tegra->domain) { err = -ENOMEM; diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index 1237df157e05..623768100c6a 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -60,8 +60,16 @@ static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, /* * If we've manually mapped the buffer object through the IOMMU, make * sure to return the IOVA address of our mapping. + * + * Similarly, for buffers that have been allocated by the DMA API the + * physical address can be used for devices that are not attached to + * an IOMMU. For these devices, callers must pass a valid pointer via + * the @phys argument. + * + * Imported buffers were also already mapped at import time, so the + * existing mapping can be reused. */ - if (phys && obj->mm) { + if (phys) { *phys = obj->iova; return NULL; } diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c index cadcdd9ea427..9ccfb56e9b01 100644 --- a/drivers/gpu/drm/tegra/plane.c +++ b/drivers/gpu/drm/tegra/plane.c @@ -3,6 +3,8 @@ * Copyright (C) 2017 NVIDIA CORPORATION. All rights reserved. */ +#include <linux/iommu.h> + #include <drm/drm_atomic.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_fourcc.h> @@ -107,21 +109,27 @@ const struct drm_plane_funcs tegra_plane_funcs = { static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state) { + struct iommu_domain *domain = iommu_get_domain_for_dev(dc->dev); unsigned int i; int err; for (i = 0; i < state->base.fb->format->num_planes; i++) { struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); + dma_addr_t phys_addr, *phys; + struct sg_table *sgt; - if (!dc->client.group) { - struct sg_table *sgt; + if (!domain || dc->client.group) + phys = &phys_addr; + else + phys = NULL; - sgt = host1x_bo_pin(dc->dev, &bo->base, NULL); - if (IS_ERR(sgt)) { - err = PTR_ERR(sgt); - goto unpin; - } + sgt = host1x_bo_pin(dc->dev, &bo->base, phys); + if (IS_ERR(sgt)) { + err = PTR_ERR(sgt); + goto unpin; + } + if (sgt) { err = dma_map_sg(dc->dev, sgt->sgl, sgt->nents, DMA_TO_DEVICE); if (err == 0) { @@ -143,7 +151,7 @@ static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state) state->iova[i] = sg_dma_address(sgt->sgl); state->sgt[i] = sgt; } else { - state->iova[i] = bo->iova; + state->iova[i] = phys_addr; } } @@ -156,9 +164,11 @@ unpin: struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); struct sg_table *sgt = state->sgt[i]; - dma_unmap_sg(dc->dev, sgt->sgl, sgt->nents, DMA_TO_DEVICE); - host1x_bo_unpin(dc->dev, &bo->base, sgt); + if (sgt) + dma_unmap_sg(dc->dev, sgt->sgl, sgt->nents, + DMA_TO_DEVICE); + host1x_bo_unpin(dc->dev, &bo->base, sgt); state->iova[i] = DMA_MAPPING_ERROR; state->sgt[i] = NULL; } @@ -172,17 +182,13 @@ static void tegra_dc_unpin(struct tegra_dc *dc, struct tegra_plane_state *state) for (i = 0; i < state->base.fb->format->num_planes; i++) { struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); + struct sg_table *sgt = state->sgt[i]; - if (!dc->client.group) { - struct sg_table *sgt = state->sgt[i]; - - if (sgt) { - dma_unmap_sg(dc->dev, sgt->sgl, sgt->nents, - DMA_TO_DEVICE); - host1x_bo_unpin(dc->dev, &bo->base, sgt); - } - } + if (sgt) + dma_unmap_sg(dc->dev, sgt->sgl, sgt->nents, + DMA_TO_DEVICE); + host1x_bo_unpin(dc->dev, &bo->base, sgt); state->iova[i] = DMA_MAPPING_ERROR; state->sgt[i] = NULL; } diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c index 41d24949478e..81226a4953c1 100644 --- a/drivers/gpu/drm/tegra/sor.c +++ b/drivers/gpu/drm/tegra/sor.c @@ -3915,6 +3915,17 @@ static int tegra_sor_probe(struct platform_device *pdev) platform_set_drvdata(pdev, sor); pm_runtime_enable(&pdev->dev); + INIT_LIST_HEAD(&sor->client.list); + sor->client.ops = &sor_client_ops; + sor->client.dev = &pdev->dev; + + err = host1x_client_register(&sor->client); + if (err < 0) { + dev_err(&pdev->dev, "failed to register host1x client: %d\n", + err); + goto rpm_disable; + } + /* * On Tegra210 and earlier, provide our own implementation for the * pad output clock. @@ -3922,16 +3933,17 @@ static int tegra_sor_probe(struct platform_device *pdev) if (!sor->clk_pad) { char *name; + name = devm_kasprintf(sor->dev, GFP_KERNEL, "sor%u_pad_clkout", + sor->index); + if (!name) { + err = -ENOMEM; + goto unregister; + } + err = host1x_client_resume(&sor->client); if (err < 0) { dev_err(sor->dev, "failed to resume: %d\n", err); - goto remove; - } - - name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "sor%u_pad_clkout", sor->index); - if (!name) { - err = -ENOMEM; - goto remove; + goto unregister; } sor->clk_pad = tegra_clk_sor_pad_register(sor, name); @@ -3940,24 +3952,17 @@ static int tegra_sor_probe(struct platform_device *pdev) if (IS_ERR(sor->clk_pad)) { err = PTR_ERR(sor->clk_pad); - dev_err(&pdev->dev, "failed to register SOR pad clock: %d\n", - err); - goto remove; - } - - INIT_LIST_HEAD(&sor->client.list); - sor->client.ops = &sor_client_ops; - sor->client.dev = &pdev->dev; - - err = host1x_client_register(&sor->client); - if (err < 0) { - dev_err(&pdev->dev, "failed to register host1x client: %d\n", + dev_err(sor->dev, "failed to register SOR pad clock: %d\n", err); - goto remove; + goto unregister; } return 0; +unregister: + host1x_client_unregister(&sor->client); +rpm_disable: + pm_runtime_disable(&pdev->dev); remove: if (sor->ops && sor->ops->remove) sor->ops->remove(sor); @@ -3971,8 +3976,6 @@ static int tegra_sor_remove(struct platform_device *pdev) struct tegra_sor *sor = platform_get_drvdata(pdev); int err; - pm_runtime_disable(&pdev->dev); - err = host1x_client_unregister(&sor->client); if (err < 0) { dev_err(&pdev->dev, "failed to unregister host1x client: %d\n", @@ -3980,6 +3983,8 @@ static int tegra_sor_remove(struct platform_device *pdev) return err; } + pm_runtime_disable(&pdev->dev); + if (sor->ops && sor->ops->remove) { err = sor->ops->remove(sor); if (err < 0) diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index eebb4c06c04d..389128b8c4dd 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -179,7 +179,6 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, pgoff_t num_prefault) { struct vm_area_struct *vma = vmf->vma; - struct vm_area_struct cvma = *vma; struct ttm_buffer_object *bo = vma->vm_private_data; struct ttm_bo_device *bdev = bo->bdev; unsigned long page_offset; @@ -250,7 +249,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, goto out_io_unlock; } - cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, prot); + prot = ttm_io_prot(bo->mem.placement, prot); if (!bo->mem.bus.is_iomem) { struct ttm_operation_ctx ctx = { .interruptible = false, @@ -266,7 +265,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, } } else { /* Iomem should not be marked encrypted */ - cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); + prot = pgprot_decrypted(prot); } /* @@ -289,11 +288,20 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, pfn = page_to_pfn(page); } + /* + * Note that the value of @prot at this point may differ from + * the value of @vma->vm_page_prot in the caching- and + * encryption bits. This is because the exact location of the + * data may not be known at mmap() time and may also change + * at arbitrary times while the data is mmap'ed. + * See vmf_insert_mixed_prot() for a discussion. + */ if (vma->vm_flags & VM_MIXEDMAP) - ret = vmf_insert_mixed(&cvma, address, - __pfn_to_pfn_t(pfn, PFN_DEV)); + ret = vmf_insert_mixed_prot(vma, address, + __pfn_to_pfn_t(pfn, PFN_DEV), + prot); else - ret = vmf_insert_pfn(&cvma, address, pfn); + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); /* Never error on prefaulted PTEs */ if (unlikely((ret & VM_FAULT_ERROR))) { @@ -325,7 +333,7 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) if (ret) return ret; - prot = vm_get_page_prot(vma->vm_flags); + prot = vma->vm_page_prot; ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) return ret; diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c index 60b2fedd0061..a10643aa89aa 100644 --- a/drivers/gpu/host1x/job.c +++ b/drivers/gpu/host1x/job.c @@ -8,6 +8,7 @@ #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/host1x.h> +#include <linux/iommu.h> #include <linux/kref.h> #include <linux/module.h> #include <linux/scatterlist.h> @@ -101,9 +102,11 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) { struct host1x_client *client = job->client; struct device *dev = client->dev; + struct iommu_domain *domain; unsigned int i; int err; + domain = iommu_get_domain_for_dev(dev); job->num_unpins = 0; for (i = 0; i < job->num_relocs; i++) { @@ -117,7 +120,19 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto unpin; } - if (client->group) + /* + * If the client device is not attached to an IOMMU, the + * physical address of the buffer object can be used. + * + * Similarly, when an IOMMU domain is shared between all + * host1x clients, the IOVA is already available, so no + * need to map the buffer object again. + * + * XXX Note that this isn't always safe to do because it + * relies on an assumption that no cache maintenance is + * needed on the buffer objects. + */ + if (!domain || client->group) phys = &phys_addr; else phys = NULL; @@ -176,6 +191,7 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) dma_addr_t phys_addr; unsigned long shift; struct iova *alloc; + dma_addr_t *phys; unsigned int j; g->bo = host1x_bo_get(g->bo); @@ -184,7 +200,17 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto unpin; } - sgt = host1x_bo_pin(host->dev, g->bo, NULL); + /** + * If the host1x is not attached to an IOMMU, there is no need + * to map the buffer object for the host1x, since the physical + * address can simply be used. + */ + if (!iommu_get_domain_for_dev(host->dev)) + phys = &phys_addr; + else + phys = NULL; + + sgt = host1x_bo_pin(host->dev, g->bo, phys); if (IS_ERR(sgt)) { err = PTR_ERR(sgt); goto unpin; @@ -214,7 +240,7 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) job->unpins[job->num_unpins].size = gather_size; phys_addr = iova_dma_addr(&host->iova, alloc); - } else { + } else if (sgt) { err = dma_map_sg(host->dev, sgt->sgl, sgt->nents, DMA_TO_DEVICE); if (!err) { @@ -222,6 +248,7 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto unpin; } + job->unpins[job->num_unpins].dir = DMA_TO_DEVICE; job->unpins[job->num_unpins].dev = host->dev; phys_addr = sg_dma_address(sgt->sgl); } @@ -229,7 +256,6 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) job->addr_phys[job->num_unpins] = phys_addr; job->gather_addr_phys[i] = phys_addr; - job->unpins[job->num_unpins].dir = DMA_TO_DEVICE; job->unpins[job->num_unpins].bo = g->bo; job->unpins[job->num_unpins].sgt = sgt; job->num_unpins++; diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 8eb167540b4f..0370364169c4 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -1351,6 +1351,8 @@ channel_message_table[CHANNELMSG_COUNT] = { { CHANNELMSG_19, 0, NULL }, { CHANNELMSG_20, 0, NULL }, { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL }, + { CHANNELMSG_22, 0, NULL }, + { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL }, }; /* @@ -1362,25 +1364,16 @@ void vmbus_onmessage(void *context) { struct hv_message *msg = context; struct vmbus_channel_message_header *hdr; - int size; hdr = (struct vmbus_channel_message_header *)msg->u.payload; - size = msg->header.payload_size; trace_vmbus_on_message(hdr); - if (hdr->msgtype >= CHANNELMSG_COUNT) { - pr_err("Received invalid channel message type %d size %d\n", - hdr->msgtype, size); - print_hex_dump_bytes("", DUMP_PREFIX_NONE, - (unsigned char *)msg->u.payload, size); - return; - } - - if (channel_message_table[hdr->msgtype].message_handler) - channel_message_table[hdr->msgtype].message_handler(hdr); - else - pr_err("Unhandled channel message type %d\n", hdr->msgtype); + /* + * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go + * out of bound and the message_handler pointer can not be NULL. + */ + channel_message_table[hdr->msgtype].message_handler(hdr); } /* diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c index b155d0052981..a02ce43d778d 100644 --- a/drivers/hv/hv_balloon.c +++ b/drivers/hv/hv_balloon.c @@ -1217,10 +1217,7 @@ static unsigned int alloc_balloon_pages(struct hv_dynmem_device *dm, unsigned int i, j; struct page *pg; - if (num_pages < alloc_unit) - return 0; - - for (i = 0; (i * alloc_unit) < num_pages; i++) { + for (i = 0; i < num_pages / alloc_unit; i++) { if (bl_resp->hdr.size + sizeof(union dm_mem_page_range) > HV_HYP_PAGE_SIZE) return i * alloc_unit; @@ -1258,7 +1255,7 @@ static unsigned int alloc_balloon_pages(struct hv_dynmem_device *dm, } - return num_pages; + return i * alloc_unit; } static void balloon_up(struct work_struct *dummy) @@ -1273,9 +1270,6 @@ static void balloon_up(struct work_struct *dummy) long avail_pages; unsigned long floor; - /* The host balloons pages in 2M granularity. */ - WARN_ON_ONCE(num_pages % PAGES_IN_2M != 0); - /* * We will attempt 2M allocations. However, if we fail to * allocate 2M chunks, we will go back to PAGE_SIZE allocations. @@ -1285,14 +1279,13 @@ static void balloon_up(struct work_struct *dummy) avail_pages = si_mem_available(); floor = compute_balloon_floor(); - /* Refuse to balloon below the floor, keep the 2M granularity. */ + /* Refuse to balloon below the floor. */ if (avail_pages < num_pages || avail_pages - num_pages < floor) { pr_warn("Balloon request will be partially fulfilled. %s\n", avail_pages < num_pages ? "Not enough memory." : "Balloon floor reached."); num_pages = avail_pages > floor ? (avail_pages - floor) : 0; - num_pages -= num_pages % PAGES_IN_2M; } while (!done) { diff --git a/drivers/hv/hv_fcopy.c b/drivers/hv/hv_fcopy.c index 08fa4a5de644..bb9ba3f7c794 100644 --- a/drivers/hv/hv_fcopy.c +++ b/drivers/hv/hv_fcopy.c @@ -346,9 +346,61 @@ int hv_fcopy_init(struct hv_util_service *srv) return 0; } +static void hv_fcopy_cancel_work(void) +{ + cancel_delayed_work_sync(&fcopy_timeout_work); + cancel_work_sync(&fcopy_send_work); +} + +int hv_fcopy_pre_suspend(void) +{ + struct vmbus_channel *channel = fcopy_transaction.recv_channel; + struct hv_fcopy_hdr *fcopy_msg; + + /* + * Fake a CANCEL_FCOPY message for the user space daemon in case the + * daemon is in the middle of copying some file. It doesn't matter if + * there is already a message pending to be delivered to the user + * space since we force fcopy_transaction.state to be HVUTIL_READY, so + * the user space daemon's write() will fail with EINVAL (see + * fcopy_on_msg()), and the daemon will reset the device by closing + * and re-opening it. + */ + fcopy_msg = kzalloc(sizeof(*fcopy_msg), GFP_KERNEL); + if (!fcopy_msg) + return -ENOMEM; + + tasklet_disable(&channel->callback_event); + + fcopy_msg->operation = CANCEL_FCOPY; + + hv_fcopy_cancel_work(); + + /* We don't care about the return value. */ + hvutil_transport_send(hvt, fcopy_msg, sizeof(*fcopy_msg), NULL); + + kfree(fcopy_msg); + + fcopy_transaction.state = HVUTIL_READY; + + /* tasklet_enable() will be called in hv_fcopy_pre_resume(). */ + return 0; +} + +int hv_fcopy_pre_resume(void) +{ + struct vmbus_channel *channel = fcopy_transaction.recv_channel; + + tasklet_enable(&channel->callback_event); + + return 0; +} + void hv_fcopy_deinit(void) { fcopy_transaction.state = HVUTIL_DEVICE_DYING; - cancel_delayed_work_sync(&fcopy_timeout_work); + + hv_fcopy_cancel_work(); + hvutil_transport_destroy(hvt); } diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c index ae7c028dc5a8..e74b144b8f3d 100644 --- a/drivers/hv/hv_kvp.c +++ b/drivers/hv/hv_kvp.c @@ -758,11 +758,50 @@ hv_kvp_init(struct hv_util_service *srv) return 0; } -void hv_kvp_deinit(void) +static void hv_kvp_cancel_work(void) { - kvp_transaction.state = HVUTIL_DEVICE_DYING; cancel_delayed_work_sync(&kvp_host_handshake_work); cancel_delayed_work_sync(&kvp_timeout_work); cancel_work_sync(&kvp_sendkey_work); +} + +int hv_kvp_pre_suspend(void) +{ + struct vmbus_channel *channel = kvp_transaction.recv_channel; + + tasklet_disable(&channel->callback_event); + + /* + * If there is a pending transtion, it's unnecessary to tell the host + * that the transaction will fail, because that is implied when + * util_suspend() calls vmbus_close() later. + */ + hv_kvp_cancel_work(); + + /* + * Forece the state to READY to handle the ICMSGTYPE_NEGOTIATE message + * later. The user space daemon may go out of order and its write() + * may fail with EINVAL: this doesn't matter since the daemon will + * reset the device by closing and re-opening it. + */ + kvp_transaction.state = HVUTIL_READY; + return 0; +} + +int hv_kvp_pre_resume(void) +{ + struct vmbus_channel *channel = kvp_transaction.recv_channel; + + tasklet_enable(&channel->callback_event); + + return 0; +} + +void hv_kvp_deinit(void) +{ + kvp_transaction.state = HVUTIL_DEVICE_DYING; + + hv_kvp_cancel_work(); + hvutil_transport_destroy(hvt); } diff --git a/drivers/hv/hv_snapshot.c b/drivers/hv/hv_snapshot.c index 03b6454268b3..1c75b38f0d6d 100644 --- a/drivers/hv/hv_snapshot.c +++ b/drivers/hv/hv_snapshot.c @@ -379,10 +379,61 @@ hv_vss_init(struct hv_util_service *srv) return 0; } -void hv_vss_deinit(void) +static void hv_vss_cancel_work(void) { - vss_transaction.state = HVUTIL_DEVICE_DYING; cancel_delayed_work_sync(&vss_timeout_work); cancel_work_sync(&vss_handle_request_work); +} + +int hv_vss_pre_suspend(void) +{ + struct vmbus_channel *channel = vss_transaction.recv_channel; + struct hv_vss_msg *vss_msg; + + /* + * Fake a THAW message for the user space daemon in case the daemon + * has frozen the file systems. It doesn't matter if there is already + * a message pending to be delivered to the user space since we force + * vss_transaction.state to be HVUTIL_READY, so the user space daemon's + * write() will fail with EINVAL (see vss_on_msg()), and the daemon + * will reset the device by closing and re-opening it. + */ + vss_msg = kzalloc(sizeof(*vss_msg), GFP_KERNEL); + if (!vss_msg) + return -ENOMEM; + + tasklet_disable(&channel->callback_event); + + vss_msg->vss_hdr.operation = VSS_OP_THAW; + + /* Cancel any possible pending work. */ + hv_vss_cancel_work(); + + /* We don't care about the return value. */ + hvutil_transport_send(hvt, vss_msg, sizeof(*vss_msg), NULL); + + kfree(vss_msg); + + vss_transaction.state = HVUTIL_READY; + + /* tasklet_enable() will be called in hv_vss_pre_resume(). */ + return 0; +} + +int hv_vss_pre_resume(void) +{ + struct vmbus_channel *channel = vss_transaction.recv_channel; + + tasklet_enable(&channel->callback_event); + + return 0; +} + +void hv_vss_deinit(void) +{ + vss_transaction.state = HVUTIL_DEVICE_DYING; + + hv_vss_cancel_work(); + hvutil_transport_destroy(hvt); } diff --git a/drivers/hv/hv_util.c b/drivers/hv/hv_util.c index 296f9098c9e4..92ee0fe4c919 100644 --- a/drivers/hv/hv_util.c +++ b/drivers/hv/hv_util.c @@ -24,6 +24,10 @@ #define SD_MAJOR 3 #define SD_MINOR 0 +#define SD_MINOR_1 1 +#define SD_MINOR_2 2 +#define SD_VERSION_3_1 (SD_MAJOR << 16 | SD_MINOR_1) +#define SD_VERSION_3_2 (SD_MAJOR << 16 | SD_MINOR_2) #define SD_VERSION (SD_MAJOR << 16 | SD_MINOR) #define SD_MAJOR_1 1 @@ -50,8 +54,10 @@ static int sd_srv_version; static int ts_srv_version; static int hb_srv_version; -#define SD_VER_COUNT 2 +#define SD_VER_COUNT 4 static const int sd_versions[] = { + SD_VERSION_3_2, + SD_VERSION_3_1, SD_VERSION, SD_VERSION_1 }; @@ -75,18 +81,56 @@ static const int fw_versions[] = { UTIL_WS2K8_FW_VERSION }; +/* + * Send the "hibernate" udev event in a thread context. + */ +struct hibernate_work_context { + struct work_struct work; + struct hv_device *dev; +}; + +static struct hibernate_work_context hibernate_context; +static bool hibernation_supported; + +static void send_hibernate_uevent(struct work_struct *work) +{ + char *uevent_env[2] = { "EVENT=hibernate", NULL }; + struct hibernate_work_context *ctx; + + ctx = container_of(work, struct hibernate_work_context, work); + + kobject_uevent_env(&ctx->dev->device.kobj, KOBJ_CHANGE, uevent_env); + + pr_info("Sent hibernation uevent\n"); +} + +static int hv_shutdown_init(struct hv_util_service *srv) +{ + struct vmbus_channel *channel = srv->channel; + + INIT_WORK(&hibernate_context.work, send_hibernate_uevent); + hibernate_context.dev = channel->device_obj; + + hibernation_supported = hv_is_hibernation_supported(); + + return 0; +} + static void shutdown_onchannelcallback(void *context); static struct hv_util_service util_shutdown = { .util_cb = shutdown_onchannelcallback, + .util_init = hv_shutdown_init, }; static int hv_timesync_init(struct hv_util_service *srv); +static int hv_timesync_pre_suspend(void); static void hv_timesync_deinit(void); static void timesync_onchannelcallback(void *context); static struct hv_util_service util_timesynch = { .util_cb = timesync_onchannelcallback, .util_init = hv_timesync_init, + .util_pre_suspend = hv_timesync_pre_suspend, .util_deinit = hv_timesync_deinit, }; @@ -98,18 +142,24 @@ static struct hv_util_service util_heartbeat = { static struct hv_util_service util_kvp = { .util_cb = hv_kvp_onchannelcallback, .util_init = hv_kvp_init, + .util_pre_suspend = hv_kvp_pre_suspend, + .util_pre_resume = hv_kvp_pre_resume, .util_deinit = hv_kvp_deinit, }; static struct hv_util_service util_vss = { .util_cb = hv_vss_onchannelcallback, .util_init = hv_vss_init, + .util_pre_suspend = hv_vss_pre_suspend, + .util_pre_resume = hv_vss_pre_resume, .util_deinit = hv_vss_deinit, }; static struct hv_util_service util_fcopy = { .util_cb = hv_fcopy_onchannelcallback, .util_init = hv_fcopy_init, + .util_pre_suspend = hv_fcopy_pre_suspend, + .util_pre_resume = hv_fcopy_pre_resume, .util_deinit = hv_fcopy_deinit, }; @@ -118,17 +168,27 @@ static void perform_shutdown(struct work_struct *dummy) orderly_poweroff(true); } +static void perform_restart(struct work_struct *dummy) +{ + orderly_reboot(); +} + /* * Perform the shutdown operation in a thread context. */ static DECLARE_WORK(shutdown_work, perform_shutdown); +/* + * Perform the restart operation in a thread context. + */ +static DECLARE_WORK(restart_work, perform_restart); + static void shutdown_onchannelcallback(void *context) { struct vmbus_channel *channel = context; + struct work_struct *work = NULL; u32 recvlen; u64 requestid; - bool execute_shutdown = false; u8 *shut_txf_buf = util_shutdown.recv_buffer; struct shutdown_msg_data *shutdown_msg; @@ -157,19 +217,37 @@ static void shutdown_onchannelcallback(void *context) sizeof(struct vmbuspipe_hdr) + sizeof(struct icmsg_hdr)]; + /* + * shutdown_msg->flags can be 0(shut down), 2(reboot), + * or 4(hibernate). It may bitwise-OR 1, which means + * performing the request by force. Linux always tries + * to perform the request by force. + */ switch (shutdown_msg->flags) { case 0: case 1: icmsghdrp->status = HV_S_OK; - execute_shutdown = true; - + work = &shutdown_work; pr_info("Shutdown request received -" " graceful shutdown initiated\n"); break; + case 2: + case 3: + icmsghdrp->status = HV_S_OK; + work = &restart_work; + pr_info("Restart request received -" + " graceful restart initiated\n"); + break; + case 4: + case 5: + pr_info("Hibernation request received\n"); + icmsghdrp->status = hibernation_supported ? + HV_S_OK : HV_E_FAIL; + if (hibernation_supported) + work = &hibernate_context.work; + break; default: icmsghdrp->status = HV_E_FAIL; - execute_shutdown = false; - pr_info("Shutdown request received -" " Invalid request\n"); break; @@ -184,8 +262,8 @@ static void shutdown_onchannelcallback(void *context) VM_PKT_DATA_INBAND, 0); } - if (execute_shutdown == true) - schedule_work(&shutdown_work); + if (work) + schedule_work(work); } /* @@ -441,6 +519,44 @@ static int util_remove(struct hv_device *dev) return 0; } +/* + * When we're in util_suspend(), all the userspace processes have been frozen + * (refer to hibernate() -> freeze_processes()). The userspace is thawed only + * after the whole resume procedure, including util_resume(), finishes. + */ +static int util_suspend(struct hv_device *dev) +{ + struct hv_util_service *srv = hv_get_drvdata(dev); + int ret = 0; + + if (srv->util_pre_suspend) { + ret = srv->util_pre_suspend(); + if (ret) + return ret; + } + + vmbus_close(dev->channel); + + return 0; +} + +static int util_resume(struct hv_device *dev) +{ + struct hv_util_service *srv = hv_get_drvdata(dev); + int ret = 0; + + if (srv->util_pre_resume) { + ret = srv->util_pre_resume(); + if (ret) + return ret; + } + + ret = vmbus_open(dev->channel, 4 * HV_HYP_PAGE_SIZE, + 4 * HV_HYP_PAGE_SIZE, NULL, 0, srv->util_cb, + dev->channel); + return ret; +} + static const struct hv_vmbus_device_id id_table[] = { /* Shutdown guid */ { HV_SHUTDOWN_GUID, @@ -477,6 +593,8 @@ static struct hv_driver util_drv = { .id_table = id_table, .probe = util_probe, .remove = util_remove, + .suspend = util_suspend, + .resume = util_resume, .driver = { .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, @@ -546,11 +664,23 @@ static int hv_timesync_init(struct hv_util_service *srv) return 0; } +static void hv_timesync_cancel_work(void) +{ + cancel_work_sync(&adj_time_work); +} + +static int hv_timesync_pre_suspend(void) +{ + hv_timesync_cancel_work(); + return 0; +} + static void hv_timesync_deinit(void) { if (hv_ptp_clock) ptp_clock_unregister(hv_ptp_clock); - cancel_work_sync(&adj_time_work); + + hv_timesync_cancel_work(); } static int __init init_hyperv_utils(void) diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 20edcfd3b96c..f5fa3b3c9baf 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -352,14 +352,20 @@ void vmbus_on_msg_dpc(unsigned long data); int hv_kvp_init(struct hv_util_service *srv); void hv_kvp_deinit(void); +int hv_kvp_pre_suspend(void); +int hv_kvp_pre_resume(void); void hv_kvp_onchannelcallback(void *context); int hv_vss_init(struct hv_util_service *srv); void hv_vss_deinit(void); +int hv_vss_pre_suspend(void); +int hv_vss_pre_resume(void); void hv_vss_onchannelcallback(void *context); int hv_fcopy_init(struct hv_util_service *srv); void hv_fcopy_deinit(void); +int hv_fcopy_pre_suspend(void); +int hv_fcopy_pre_resume(void); void hv_fcopy_onchannelcallback(void *context); void vmbus_initiate_unload(bool crash); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 4ef5a66df680..029378c27421 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1033,6 +1033,10 @@ void vmbus_on_msg_dpc(unsigned long data) } entry = &channel_message_table[hdr->msgtype]; + + if (!entry->message_handler) + goto msg_handled; + if (entry->handler_type == VMHT_BLOCKING) { ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC); if (ctx == NULL) diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 17583bf8c2dc..d4c83009d625 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -595,19 +595,18 @@ static int i8k_open_fs(struct inode *inode, struct file *file) return single_open(file, i8k_proc_show, NULL); } -static const struct file_operations i8k_fops = { - .owner = THIS_MODULE, - .open = i8k_open_fs, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .unlocked_ioctl = i8k_ioctl, +static const struct proc_ops i8k_proc_ops = { + .proc_open = i8k_open_fs, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_ioctl = i8k_ioctl, }; static void __init i8k_init_procfs(void) { /* Register the proc entry */ - proc_create("i8k", 0, NULL, &i8k_fops); + proc_create("i8k", 0, NULL, &i8k_proc_ops); } static void __exit i8k_exit_procfs(void) diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 14e1a532ebb5..3b05560456ea 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -76,7 +76,6 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) struct device_node *node = pdev->dev.of_node; struct hwspinlock_device *bank; struct hwspinlock *hwlock; - struct resource *res; void __iomem *io_base; int num_locks, i, ret; /* Only a single hwspinlock block device is supported */ @@ -85,13 +84,9 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) if (!node) return -ENODEV; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - - io_base = ioremap(res->start, resource_size(res)); - if (!io_base) - return -ENOMEM; + io_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(io_base)) + return PTR_ERR(io_base); /* * make sure the module is enabled and clocked before reading @@ -101,7 +96,7 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) { pm_runtime_put_noidle(&pdev->dev); - goto iounmap_base; + goto runtime_err; } /* Determine number of locks */ @@ -114,20 +109,21 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) */ ret = pm_runtime_put(&pdev->dev); if (ret < 0) - goto iounmap_base; + goto runtime_err; /* one of the four lsb's must be set, and nothing else */ if (hweight_long(i & 0xf) != 1 || i > 8) { ret = -EINVAL; - goto iounmap_base; + goto runtime_err; } num_locks = i * 32; /* actual number of locks in this device */ - bank = kzalloc(struct_size(bank, lock, num_locks), GFP_KERNEL); + bank = devm_kzalloc(&pdev->dev, struct_size(bank, lock, num_locks), + GFP_KERNEL); if (!bank) { ret = -ENOMEM; - goto iounmap_base; + goto runtime_err; } platform_set_drvdata(pdev, bank); @@ -138,25 +134,21 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) ret = hwspin_lock_register(bank, &pdev->dev, &omap_hwspinlock_ops, base_id, num_locks); if (ret) - goto reg_fail; + goto runtime_err; dev_dbg(&pdev->dev, "Registered %d locks with HwSpinlock core\n", num_locks); return 0; -reg_fail: - kfree(bank); -iounmap_base: +runtime_err: pm_runtime_disable(&pdev->dev); - iounmap(io_base); return ret; } static int omap_hwspinlock_remove(struct platform_device *pdev) { struct hwspinlock_device *bank = platform_get_drvdata(pdev); - void __iomem *io_base = bank->lock[0].priv - LOCK_BASE_OFFSET; int ret; ret = hwspin_lock_unregister(bank); @@ -166,8 +158,6 @@ static int omap_hwspinlock_remove(struct platform_device *pdev) } pm_runtime_disable(&pdev->dev); - iounmap(io_base); - kfree(bank); return 0; } diff --git a/drivers/hwspinlock/qcom_hwspinlock.c b/drivers/hwspinlock/qcom_hwspinlock.c index 6da7447d277d..f0da544b14d2 100644 --- a/drivers/hwspinlock/qcom_hwspinlock.c +++ b/drivers/hwspinlock/qcom_hwspinlock.c @@ -12,7 +12,6 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/platform_device.h> -#include <linux/pm_runtime.h> #include <linux/regmap.h> #include "hwspinlock_internal.h" @@ -122,35 +121,12 @@ static int qcom_hwspinlock_probe(struct platform_device *pdev) regmap, field); } - pm_runtime_enable(&pdev->dev); - - ret = hwspin_lock_register(bank, &pdev->dev, &qcom_hwspinlock_ops, - 0, QCOM_MUTEX_NUM_LOCKS); - if (ret) - pm_runtime_disable(&pdev->dev); - - return ret; -} - -static int qcom_hwspinlock_remove(struct platform_device *pdev) -{ - struct hwspinlock_device *bank = platform_get_drvdata(pdev); - int ret; - - ret = hwspin_lock_unregister(bank); - if (ret) { - dev_err(&pdev->dev, "%s failed: %d\n", __func__, ret); - return ret; - } - - pm_runtime_disable(&pdev->dev); - - return 0; + return devm_hwspin_lock_register(&pdev->dev, bank, &qcom_hwspinlock_ops, + 0, QCOM_MUTEX_NUM_LOCKS); } static struct platform_driver qcom_hwspinlock_driver = { .probe = qcom_hwspinlock_probe, - .remove = qcom_hwspinlock_remove, .driver = { .name = "qcom_hwspinlock", .of_match_table = qcom_hwspinlock_of_match, diff --git a/drivers/hwspinlock/sirf_hwspinlock.c b/drivers/hwspinlock/sirf_hwspinlock.c index 1f625cd68c50..823d3c4f621e 100644 --- a/drivers/hwspinlock/sirf_hwspinlock.c +++ b/drivers/hwspinlock/sirf_hwspinlock.c @@ -9,7 +9,6 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/io.h> -#include <linux/pm_runtime.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/hwspinlock.h> @@ -56,7 +55,7 @@ static int sirf_hwspinlock_probe(struct platform_device *pdev) { struct sirf_hwspinlock *hwspin; struct hwspinlock *hwlock; - int idx, ret; + int idx; if (!pdev->dev.of_node) return -ENODEV; @@ -69,9 +68,9 @@ static int sirf_hwspinlock_probe(struct platform_device *pdev) return -ENOMEM; /* retrieve io base */ - hwspin->io_base = of_iomap(pdev->dev.of_node, 0); - if (!hwspin->io_base) - return -ENOMEM; + hwspin->io_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hwspin->io_base)) + return PTR_ERR(hwspin->io_base); for (idx = 0; idx < HW_SPINLOCK_NUMBER; idx++) { hwlock = &hwspin->bank.lock[idx]; @@ -80,39 +79,9 @@ static int sirf_hwspinlock_probe(struct platform_device *pdev) platform_set_drvdata(pdev, hwspin); - pm_runtime_enable(&pdev->dev); - - ret = hwspin_lock_register(&hwspin->bank, &pdev->dev, - &sirf_hwspinlock_ops, 0, - HW_SPINLOCK_NUMBER); - if (ret) - goto reg_failed; - - return 0; - -reg_failed: - pm_runtime_disable(&pdev->dev); - iounmap(hwspin->io_base); - - return ret; -} - -static int sirf_hwspinlock_remove(struct platform_device *pdev) -{ - struct sirf_hwspinlock *hwspin = platform_get_drvdata(pdev); - int ret; - - ret = hwspin_lock_unregister(&hwspin->bank); - if (ret) { - dev_err(&pdev->dev, "%s failed: %d\n", __func__, ret); - return ret; - } - - pm_runtime_disable(&pdev->dev); - - iounmap(hwspin->io_base); - - return 0; + return devm_hwspin_lock_register(&pdev->dev, &hwspin->bank, + &sirf_hwspinlock_ops, 0, + HW_SPINLOCK_NUMBER); } static const struct of_device_id sirf_hwpinlock_ids[] = { @@ -123,7 +92,6 @@ MODULE_DEVICE_TABLE(of, sirf_hwpinlock_ids); static struct platform_driver sirf_hwspinlock_driver = { .probe = sirf_hwspinlock_probe, - .remove = sirf_hwspinlock_remove, .driver = { .name = "atlas7_hwspinlock", .of_match_table = of_match_ptr(sirf_hwpinlock_ids), diff --git a/drivers/hwspinlock/stm32_hwspinlock.c b/drivers/hwspinlock/stm32_hwspinlock.c index c8eacf4f9692..3ad0ce0da4d9 100644 --- a/drivers/hwspinlock/stm32_hwspinlock.c +++ b/drivers/hwspinlock/stm32_hwspinlock.c @@ -58,12 +58,10 @@ static int stm32_hwspinlock_probe(struct platform_device *pdev) { struct stm32_hwspinlock *hw; void __iomem *io_base; - struct resource *res; size_t array_size; int i, ret; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - io_base = devm_ioremap_resource(&pdev->dev, res); + io_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(io_base)) return PTR_ERR(io_base); diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c index a5a95ea5b81a..febb7c7ea72b 100644 --- a/drivers/i2c/busses/i2c-mv64xxx.c +++ b/drivers/i2c/busses/i2c-mv64xxx.c @@ -901,14 +901,13 @@ mv64xxx_i2c_probe(struct platform_device *pd) /* Not all platforms have clocks */ drv_data->clk = devm_clk_get(&pd->dev, NULL); - if (IS_ERR(drv_data->clk) && PTR_ERR(drv_data->clk) == -EPROBE_DEFER) + if (PTR_ERR(drv_data->clk) == -EPROBE_DEFER) return -EPROBE_DEFER; if (!IS_ERR(drv_data->clk)) clk_prepare_enable(drv_data->clk); drv_data->reg_clk = devm_clk_get(&pd->dev, "reg"); - if (IS_ERR(drv_data->reg_clk) && - PTR_ERR(drv_data->reg_clk) == -EPROBE_DEFER) + if (PTR_ERR(drv_data->reg_clk) == -EPROBE_DEFER) return -EPROBE_DEFER; if (!IS_ERR(drv_data->reg_clk)) clk_prepare_enable(drv_data->reg_clk); diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c index 39762f0611b1..86026798b4f7 100644 --- a/drivers/i2c/busses/i2c-synquacer.c +++ b/drivers/i2c/busses/i2c-synquacer.c @@ -553,7 +553,7 @@ static int synquacer_i2c_probe(struct platform_device *pdev) &i2c->pclkrate); i2c->pclk = devm_clk_get(&pdev->dev, "pclk"); - if (IS_ERR(i2c->pclk) && PTR_ERR(i2c->pclk) == -EPROBE_DEFER) + if (PTR_ERR(i2c->pclk) == -EPROBE_DEFER) return -EPROBE_DEFER; if (!IS_ERR_OR_NULL(i2c->pclk)) { dev_dbg(&pdev->dev, "clock source %p\n", i2c->pclk); diff --git a/drivers/ide/ide-proc.c b/drivers/ide/ide-proc.c index e73016cbd406..15c17f3781ee 100644 --- a/drivers/ide/ide-proc.c +++ b/drivers/ide/ide-proc.c @@ -381,13 +381,12 @@ parse_error: return -EINVAL; } -static const struct file_operations ide_settings_proc_fops = { - .owner = THIS_MODULE, - .open = ide_settings_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = ide_settings_proc_write, +static const struct proc_ops ide_settings_proc_ops = { + .proc_open = ide_settings_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = ide_settings_proc_write, }; int ide_capacity_proc_show(struct seq_file *m, void *v) @@ -546,7 +545,7 @@ void ide_proc_port_register_devices(ide_hwif_t *hwif) if (drive->proc) { ide_add_proc_entries(drive->proc, generic_drive_entries, drive); proc_create_data("settings", S_IFREG|S_IRUSR|S_IWUSR, - drive->proc, &ide_settings_proc_fops, + drive->proc, &ide_settings_proc_ops, drive); } sprintf(name, "ide%d/%s", (drive->name[2]-'a')/2, drive->name); @@ -615,7 +614,7 @@ static int ide_drivers_show(struct seq_file *s, void *p) return 0; } -DEFINE_SHOW_ATTRIBUTE(ide_drivers); +DEFINE_PROC_SHOW_ATTRIBUTE(ide_drivers); void proc_ide_create(void) { @@ -624,7 +623,7 @@ void proc_ide_create(void) if (!proc_ide_root) return; - proc_create("drivers", 0, proc_ide_root, &ide_drivers_fops); + proc_create("drivers", 0, proc_ide_root, &ide_drivers_proc_ops); } void proc_ide_destroy(void) diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 7833e650789f..d55606608ac8 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -63,6 +63,7 @@ static struct cpuidle_driver intel_idle_driver = { }; /* intel_idle.max_cstate=0 disables driver */ static int max_cstate = CPUIDLE_STATE_MAX - 1; +static unsigned int disabled_states_mask; static unsigned int mwait_substates; @@ -1131,6 +1132,10 @@ static bool no_acpi __read_mostly; module_param(no_acpi, bool, 0444); MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); +static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ +module_param_named(use_acpi, force_use_acpi, bool, 0444); +MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); + static struct acpi_processor_power acpi_state_table __initdata; /** @@ -1230,6 +1235,9 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) if (cx->type > ACPI_STATE_C2) state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; + if (disabled_states_mask & BIT(cstate)) + state->flags |= CPUIDLE_FLAG_OFF; + state->enter = intel_idle; state->enter_s2idle = intel_idle_s2idle; } @@ -1258,6 +1266,8 @@ static bool __init intel_idle_off_by_default(u32 mwait_hint) return true; } #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ +#define force_use_acpi (false) + static inline bool intel_idle_acpi_cst_extract(void) { return false; } static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } @@ -1460,8 +1470,10 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) /* Structure copy. */ drv->states[drv->state_count] = cpuidle_state_table[cstate]; - if (icpu->use_acpi && intel_idle_off_by_default(mwait_hint) && - !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE)) + if ((disabled_states_mask & BIT(drv->state_count)) || + ((icpu->use_acpi || force_use_acpi) && + intel_idle_off_by_default(mwait_hint) && + !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF; drv->state_count++; @@ -1480,6 +1492,10 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) { cpuidle_poll_state_init(drv); + + if (disabled_states_mask & BIT(0)) + drv->states[0].flags |= CPUIDLE_FLAG_OFF; + drv->state_count = 1; if (icpu) @@ -1607,7 +1623,7 @@ static int __init intel_idle_init(void) icpu = (const struct idle_cpu *)id->driver_data; if (icpu) { cpuidle_state_table = icpu->state_table; - if (icpu->use_acpi) + if (icpu->use_acpi || force_use_acpi) intel_idle_acpi_cst_extract(); } else if (!intel_idle_acpi_cst_extract()) { return -ENODEV; @@ -1660,3 +1676,11 @@ device_initcall(intel_idle_init); * is the easiest way (currently) to continue doing that. */ module_param(max_cstate, int, 0444); +/* + * The positions of the bits that are set in this number are the indices of the + * idle states to be disabled by default (as reflected by the names of the + * corresponding idle state directories in sysfs, "state0", "state1" ... + * "state<i>" ..., where <i> is the index of the given state). + */ +module_param_named(states_off, disabled_states_mask, uint, 0444); +MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); diff --git a/drivers/iio/accel/cros_ec_accel_legacy.c b/drivers/iio/accel/cros_ec_accel_legacy.c index 65f85faf6f31..68e847c6255e 100644 --- a/drivers/iio/accel/cros_ec_accel_legacy.c +++ b/drivers/iio/accel/cros_ec_accel_legacy.c @@ -18,7 +18,6 @@ #include <linux/iio/trigger_consumer.h> #include <linux/iio/triggered_buffer.h> #include <linux/kernel.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/platform_data/cros_ec_commands.h> diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c index 7dce04473467..576e45faafaf 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c @@ -16,7 +16,6 @@ #include <linux/iio/trigger_consumer.h> #include <linux/iio/triggered_buffer.h> #include <linux/kernel.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c index 81a7f692de2f..d3a3626c7cd8 100644 --- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c +++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c @@ -13,7 +13,6 @@ #include <linux/iio/kfifo_buf.h> #include <linux/iio/trigger_consumer.h> #include <linux/kernel.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/platform_data/cros_ec_commands.h> diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c index d85a391e50c5..7a838e2956f4 100644 --- a/drivers/iio/light/cros_ec_light_prox.c +++ b/drivers/iio/light/cros_ec_light_prox.c @@ -14,7 +14,6 @@ #include <linux/iio/triggered_buffer.h> #include <linux/iio/trigger_consumer.h> #include <linux/kernel.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> diff --git a/drivers/iio/pressure/cros_ec_baro.c b/drivers/iio/pressure/cros_ec_baro.c index 52f53f3123b1..b521bebd551c 100644 --- a/drivers/iio/pressure/cros_ec_baro.c +++ b/drivers/iio/pressure/cros_ec_baro.c @@ -14,7 +14,6 @@ #include <linux/iio/triggered_buffer.h> #include <linux/iio/trigger_consumer.h> #include <linux/kernel.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/platform_data/cros_ec_commands.h> diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index 9a8871e21545..d1b14887960e 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -11,7 +11,8 @@ ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \ device.o fmr_pool.o cache.o netlink.o \ roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \ multicast.o mad.o smi.o agent.o mad_rmpp.o \ - nldev.o restrack.o counters.o ib_core_uverbs.o + nldev.o restrack.o counters.o ib_core_uverbs.o \ + trace.o ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o @@ -20,7 +21,8 @@ ib_cm-y := cm.o iw_cm-y := iwcm.o iwpm_util.o iwpm_msg.o -rdma_cm-y := cma.o +CFLAGS_cma_trace.o += -I$(src) +rdma_cm-y := cma.o cma_trace.o rdma_cm-$(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS) += cma_configfs.o @@ -33,6 +35,7 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types_cq.o \ uverbs_std_types_flow_action.o uverbs_std_types_dm.o \ uverbs_std_types_mr.o uverbs_std_types_counters.o \ - uverbs_uapi.o uverbs_std_types_device.o + uverbs_uapi.o uverbs_std_types_device.o \ + uverbs_std_types_async_fd.o ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o ib_uverbs-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 606fa6d86685..1753a9801b70 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -139,7 +139,7 @@ int ib_nl_handle_ip_res_resp(struct sk_buff *skb, if (ib_nl_is_good_ip_resp(nlh)) ib_nl_process_good_ip_rsep(nlh); - return skb->len; + return 0; } static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr, diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index d535995711c3..17bfedd24cc3 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -51,9 +51,8 @@ struct ib_pkey_cache { struct ib_update_work { struct work_struct work; - struct ib_device *device; - u8 port_num; - bool enforce_security; + struct ib_event event; + bool enforce_security; }; union ib_gid zgid; @@ -130,7 +129,7 @@ static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port) event.element.port_num = port; event.event = IB_EVENT_GID_CHANGE; - ib_dispatch_event(&event); + ib_dispatch_event_clients(&event); } static const char * const gid_type_str[] = { @@ -1034,7 +1033,7 @@ int ib_get_cached_pkey(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); cache = device->port_data[port_num].cache.pkey; @@ -1043,7 +1042,7 @@ int ib_get_cached_pkey(struct ib_device *device, else *pkey = cache->table[index]; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1058,9 +1057,9 @@ int ib_get_cached_subnet_prefix(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); *sn_pfx = device->port_data[port_num].cache.subnet_prefix; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return 0; } @@ -1080,7 +1079,7 @@ int ib_find_cached_pkey(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); cache = device->port_data[port_num].cache.pkey; @@ -1101,7 +1100,7 @@ int ib_find_cached_pkey(struct ib_device *device, ret = 0; } - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1120,7 +1119,7 @@ int ib_find_exact_cached_pkey(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); cache = device->port_data[port_num].cache.pkey; @@ -1133,7 +1132,7 @@ int ib_find_exact_cached_pkey(struct ib_device *device, break; } - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1149,9 +1148,9 @@ int ib_get_cached_lmc(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); *lmc = device->port_data[port_num].cache.lmc; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1167,9 +1166,9 @@ int ib_get_cached_port_state(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); *port_state = device->port_data[port_num].cache.port_state; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1381,9 +1380,8 @@ err: return ret; } -static void ib_cache_update(struct ib_device *device, - u8 port, - bool enforce_security) +static int +ib_cache_update(struct ib_device *device, u8 port, bool enforce_security) { struct ib_port_attr *tprops = NULL; struct ib_pkey_cache *pkey_cache = NULL, *old_pkey_cache; @@ -1391,11 +1389,11 @@ static void ib_cache_update(struct ib_device *device, int ret; if (!rdma_is_port_valid(device, port)) - return; + return -EINVAL; tprops = kmalloc(sizeof *tprops, GFP_KERNEL); if (!tprops) - return; + return -ENOMEM; ret = ib_query_port(device, port, tprops); if (ret) { @@ -1413,8 +1411,10 @@ static void ib_cache_update(struct ib_device *device, pkey_cache = kmalloc(struct_size(pkey_cache, table, tprops->pkey_tbl_len), GFP_KERNEL); - if (!pkey_cache) + if (!pkey_cache) { + ret = -ENOMEM; goto err; + } pkey_cache->table_len = tprops->pkey_tbl_len; @@ -1428,7 +1428,7 @@ static void ib_cache_update(struct ib_device *device, } } - write_lock_irq(&device->cache.lock); + write_lock_irq(&device->cache_lock); old_pkey_cache = device->port_data[port].cache.pkey; @@ -1437,7 +1437,7 @@ static void ib_cache_update(struct ib_device *device, device->port_data[port].cache.port_state = tprops->state; device->port_data[port].cache.subnet_prefix = tprops->subnet_prefix; - write_unlock_irq(&device->cache.lock); + write_unlock_irq(&device->cache_lock); if (enforce_security) ib_security_cache_change(device, @@ -1446,57 +1446,91 @@ static void ib_cache_update(struct ib_device *device, kfree(old_pkey_cache); kfree(tprops); - return; + return 0; err: kfree(pkey_cache); kfree(tprops); + return ret; +} + +static void ib_cache_event_task(struct work_struct *_work) +{ + struct ib_update_work *work = + container_of(_work, struct ib_update_work, work); + int ret; + + /* Before distributing the cache update event, first sync + * the cache. + */ + ret = ib_cache_update(work->event.device, work->event.element.port_num, + work->enforce_security); + + /* GID event is notified already for individual GID entries by + * dispatch_gid_change_event(). Hence, notifiy for rest of the + * events. + */ + if (!ret && work->event.event != IB_EVENT_GID_CHANGE) + ib_dispatch_event_clients(&work->event); + + kfree(work); } -static void ib_cache_task(struct work_struct *_work) +static void ib_generic_event_task(struct work_struct *_work) { struct ib_update_work *work = container_of(_work, struct ib_update_work, work); - ib_cache_update(work->device, - work->port_num, - work->enforce_security); + ib_dispatch_event_clients(&work->event); kfree(work); } -static void ib_cache_event(struct ib_event_handler *handler, - struct ib_event *event) +static bool is_cache_update_event(const struct ib_event *event) +{ + return (event->event == IB_EVENT_PORT_ERR || + event->event == IB_EVENT_PORT_ACTIVE || + event->event == IB_EVENT_LID_CHANGE || + event->event == IB_EVENT_PKEY_CHANGE || + event->event == IB_EVENT_CLIENT_REREGISTER || + event->event == IB_EVENT_GID_CHANGE); +} + +/** + * ib_dispatch_event - Dispatch an asynchronous event + * @event:Event to dispatch + * + * Low-level drivers must call ib_dispatch_event() to dispatch the + * event to all registered event handlers when an asynchronous event + * occurs. + */ +void ib_dispatch_event(const struct ib_event *event) { struct ib_update_work *work; - if (event->event == IB_EVENT_PORT_ERR || - event->event == IB_EVENT_PORT_ACTIVE || - event->event == IB_EVENT_LID_CHANGE || - event->event == IB_EVENT_PKEY_CHANGE || - event->event == IB_EVENT_CLIENT_REREGISTER || - event->event == IB_EVENT_GID_CHANGE) { - work = kmalloc(sizeof *work, GFP_ATOMIC); - if (work) { - INIT_WORK(&work->work, ib_cache_task); - work->device = event->device; - work->port_num = event->element.port_num; - if (event->event == IB_EVENT_PKEY_CHANGE || - event->event == IB_EVENT_GID_CHANGE) - work->enforce_security = true; - else - work->enforce_security = false; - - queue_work(ib_wq, &work->work); - } - } + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return; + + if (is_cache_update_event(event)) + INIT_WORK(&work->work, ib_cache_event_task); + else + INIT_WORK(&work->work, ib_generic_event_task); + + work->event = *event; + if (event->event == IB_EVENT_PKEY_CHANGE || + event->event == IB_EVENT_GID_CHANGE) + work->enforce_security = true; + + queue_work(ib_wq, &work->work); } +EXPORT_SYMBOL(ib_dispatch_event); int ib_cache_setup_one(struct ib_device *device) { unsigned int p; int err; - rwlock_init(&device->cache.lock); + rwlock_init(&device->cache_lock); err = gid_table_setup_one(device); if (err) @@ -1505,9 +1539,6 @@ int ib_cache_setup_one(struct ib_device *device) rdma_for_each_port (device, p) ib_cache_update(device, p, true); - INIT_IB_EVENT_HANDLER(&device->cache.event_handler, - device, ib_cache_event); - ib_register_event_handler(&device->cache.event_handler); return 0; } @@ -1529,14 +1560,12 @@ void ib_cache_release_one(struct ib_device *device) void ib_cache_cleanup_one(struct ib_device *device) { - /* The cleanup function unregisters the event handler, - * waits for all in-progress workqueue elements and cleans - * up the GID cache. This function should be called after - * the device was removed from the devices list and all - * clients were removed, so the cache exists but is + /* The cleanup function waits for all in-progress workqueue + * elements and cleans up the GID cache. This function should be + * called after the device was removed from the devices list and + * all clients were removed, so the cache exists but is * non-functional and shouldn't be updated anymore. */ - ib_unregister_event_handler(&device->cache.event_handler); flush_workqueue(ib_wq); gid_table_cleanup_one(device); diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 455b3659d84b..68cc1b2d6824 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -241,6 +241,7 @@ struct cm_id_private { /* Number of clients sharing this ib_cm_id. Only valid for listeners. * Protected by the cm.lock spinlock. */ int listen_sharecount; + struct rcu_head rcu; struct ib_mad_send_buf *msg; struct cm_timewait_info *timewait_info; @@ -593,28 +594,16 @@ static void cm_free_id(__be32 local_id) xa_erase_irq(&cm.local_id_table, cm_local_id(local_id)); } -static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id) +static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id) { struct cm_id_private *cm_id_priv; + rcu_read_lock(); cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id)); - if (cm_id_priv) { - if (cm_id_priv->id.remote_id == remote_id) - refcount_inc(&cm_id_priv->refcount); - else - cm_id_priv = NULL; - } - - return cm_id_priv; -} - -static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id) -{ - struct cm_id_private *cm_id_priv; - - spin_lock_irq(&cm.lock); - cm_id_priv = cm_get_id(local_id, remote_id); - spin_unlock_irq(&cm.lock); + if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id || + !refcount_inc_not_zero(&cm_id_priv->refcount)) + cm_id_priv = NULL; + rcu_read_unlock(); return cm_id_priv; } @@ -1089,7 +1078,7 @@ retest: rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr); rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr); kfree(cm_id_priv->private_data); - kfree(cm_id_priv); + kfree_rcu(cm_id_priv, rcu); } void ib_destroy_cm_id(struct ib_cm_id *cm_id) @@ -1262,54 +1251,72 @@ static void cm_format_req(struct cm_req_msg *req_msg, cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID, cm_form_tid(cm_id_priv)); - req_msg->local_comm_id = cm_id_priv->id.local_id; - req_msg->service_id = param->service_id; - req_msg->local_ca_guid = cm_id_priv->id.device->node_guid; - cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num)); - cm_req_set_init_depth(req_msg, param->initiator_depth); - cm_req_set_remote_resp_timeout(req_msg, - param->remote_cm_response_timeout); + IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id)); + IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg, + be64_to_cpu(cm_id_priv->id.device->node_guid)); + IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num); + IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth); + IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg, + param->remote_cm_response_timeout); cm_req_set_qp_type(req_msg, param->qp_type); - cm_req_set_flow_ctrl(req_msg, param->flow_control); - cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn)); - cm_req_set_local_resp_timeout(req_msg, - param->local_cm_response_timeout); - req_msg->pkey = param->primary_path->pkey; - cm_req_set_path_mtu(req_msg, param->primary_path->mtu); - cm_req_set_max_cm_retries(req_msg, param->max_cm_retries); + IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control); + IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn); + IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg, + param->local_cm_response_timeout); + IBA_SET(CM_REQ_PARTITION_KEY, req_msg, + be16_to_cpu(param->primary_path->pkey)); + IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg, + param->primary_path->mtu); + IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries); if (param->qp_type != IB_QPT_XRC_INI) { - cm_req_set_resp_res(req_msg, param->responder_resources); - cm_req_set_retry_count(req_msg, param->retry_count); - cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count); - cm_req_set_srq(req_msg, param->srq); + IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg, + param->responder_resources); + IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count); + IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg, + param->rnr_retry_count); + IBA_SET(CM_REQ_SRQ, req_msg, param->srq); } - req_msg->primary_local_gid = pri_path->sgid; - req_msg->primary_remote_gid = pri_path->dgid; + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) = + pri_path->sgid; + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) = + pri_path->dgid; if (pri_ext) { - req_msg->primary_local_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid)); - req_msg->primary_remote_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid)); + IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid)); + IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid)); } if (pri_path->hop_limit <= 1) { - req_msg->primary_local_lid = pri_ext ? 0 : - htons(ntohl(sa_path_get_slid(pri_path))); - req_msg->primary_remote_lid = pri_ext ? 0 : - htons(ntohl(sa_path_get_dlid(pri_path))); + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(pri_ext ? 0 : + htons(ntohl(sa_path_get_slid( + pri_path))))); + IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, + be16_to_cpu(pri_ext ? 0 : + htons(ntohl(sa_path_get_dlid( + pri_path))))); } else { /* Work-around until there's a way to obtain remote LID info */ - req_msg->primary_local_lid = IB_LID_PERMISSIVE; - req_msg->primary_remote_lid = IB_LID_PERMISSIVE; + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); + IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); } - cm_req_set_primary_flow_label(req_msg, pri_path->flow_label); - cm_req_set_primary_packet_rate(req_msg, pri_path->rate); - req_msg->primary_traffic_class = pri_path->traffic_class; - req_msg->primary_hop_limit = pri_path->hop_limit; - cm_req_set_primary_sl(req_msg, pri_path->sl); - cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1)); - cm_req_set_primary_local_ack_timeout(req_msg, + IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg, + be32_to_cpu(pri_path->flow_label)); + IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate); + IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class); + IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit); + IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl); + IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg, + (pri_path->hop_limit <= 1)); + IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg, cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, pri_path->packet_life_time)); @@ -1320,38 +1327,55 @@ static void cm_format_req(struct cm_req_msg *req_msg, alt_ext = opa_is_extended_lid(alt_path->opa.dlid, alt_path->opa.slid); - req_msg->alt_local_gid = alt_path->sgid; - req_msg->alt_remote_gid = alt_path->dgid; + *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) = + alt_path->sgid; + *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) = + alt_path->dgid; if (alt_ext) { - req_msg->alt_local_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid)); - req_msg->alt_remote_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid)); + IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, + req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid)); + IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, + req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid)); } if (alt_path->hop_limit <= 1) { - req_msg->alt_local_lid = alt_ext ? 0 : - htons(ntohl(sa_path_get_slid(alt_path))); - req_msg->alt_remote_lid = alt_ext ? 0 : - htons(ntohl(sa_path_get_dlid(alt_path))); + IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, + be16_to_cpu( + alt_ext ? 0 : + htons(ntohl(sa_path_get_slid( + alt_path))))); + IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, + be16_to_cpu( + alt_ext ? 0 : + htons(ntohl(sa_path_get_dlid( + alt_path))))); } else { - req_msg->alt_local_lid = IB_LID_PERMISSIVE; - req_msg->alt_remote_lid = IB_LID_PERMISSIVE; + IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); + IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); } - cm_req_set_alt_flow_label(req_msg, - alt_path->flow_label); - cm_req_set_alt_packet_rate(req_msg, alt_path->rate); - req_msg->alt_traffic_class = alt_path->traffic_class; - req_msg->alt_hop_limit = alt_path->hop_limit; - cm_req_set_alt_sl(req_msg, alt_path->sl); - cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1)); - cm_req_set_alt_local_ack_timeout(req_msg, + IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg, + be32_to_cpu(alt_path->flow_label)); + IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate); + IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg, + alt_path->traffic_class); + IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg, + alt_path->hop_limit); + IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl); + IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg, + (alt_path->hop_limit <= 1)); + IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg, cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, alt_path->packet_life_time)); } if (param->private_data && param->private_data_len) - memcpy(req_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data, + param->private_data_len); } static int cm_validate_req_param(struct ib_cm_req_param *param) @@ -1443,8 +1467,8 @@ int ib_send_cm_req(struct ib_cm_id *cm_id, cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms; cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT; - cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg); - cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg); + cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); + cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); spin_lock_irqsave(&cm_id_priv->lock, flags); ret = ib_post_send_mad(cm_id_priv->msg, NULL); @@ -1482,14 +1506,16 @@ static int cm_issue_rej(struct cm_port *port, rej_msg = (struct cm_rej_msg *) msg->mad; cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid); - rej_msg->remote_comm_id = rcv_msg->local_comm_id; - rej_msg->local_comm_id = rcv_msg->remote_comm_id; - cm_rej_set_msg_rejected(rej_msg, msg_rejected); - rej_msg->reason = cpu_to_be16(reason); + IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, + IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg)); + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg)); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected); + IBA_SET(CM_REJ_REASON, rej_msg, reason); if (ari && ari_length) { - cm_rej_set_reject_info_len(rej_msg, ari_length); - memcpy(rej_msg->ari, ari, ari_length); + IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); + IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); } ret = ib_post_send_mad(msg, NULL); @@ -1501,8 +1527,10 @@ static int cm_issue_rej(struct cm_port *port, static bool cm_req_has_alt_path(struct cm_req_msg *req_msg) { - return ((req_msg->alt_local_lid) || - (ib_is_opa_gid(&req_msg->alt_local_gid))); + return ((cpu_to_be16( + IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) || + (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, + req_msg)))); } static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num, @@ -1522,14 +1550,18 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) { sa_path_set_dlid(primary_path, - ntohs(req_msg->primary_local_lid)); + IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, + req_msg)); sa_path_set_slid(primary_path, - ntohs(req_msg->primary_remote_lid)); + IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, + req_msg)); } else { - lid = opa_get_lid_from_gid(&req_msg->primary_local_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)); sa_path_set_dlid(primary_path, lid); - lid = opa_get_lid_from_gid(&req_msg->primary_remote_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)); sa_path_set_slid(primary_path, lid); } @@ -1537,13 +1569,19 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, return; if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) { - sa_path_set_dlid(alt_path, ntohs(req_msg->alt_local_lid)); - sa_path_set_slid(alt_path, ntohs(req_msg->alt_remote_lid)); + sa_path_set_dlid(alt_path, + IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, + req_msg)); + sa_path_set_slid(alt_path, + IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, + req_msg)); } else { - lid = opa_get_lid_from_gid(&req_msg->alt_local_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg)); sa_path_set_dlid(alt_path, lid); - lid = opa_get_lid_from_gid(&req_msg->alt_remote_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg)); sa_path_set_slid(alt_path, lid); } } @@ -1552,44 +1590,58 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, struct sa_path_rec *primary_path, struct sa_path_rec *alt_path) { - primary_path->dgid = req_msg->primary_local_gid; - primary_path->sgid = req_msg->primary_remote_gid; - primary_path->flow_label = cm_req_get_primary_flow_label(req_msg); - primary_path->hop_limit = req_msg->primary_hop_limit; - primary_path->traffic_class = req_msg->primary_traffic_class; + primary_path->dgid = + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg); + primary_path->sgid = + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg); + primary_path->flow_label = + cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg)); + primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg); + primary_path->traffic_class = + IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg); primary_path->reversible = 1; - primary_path->pkey = req_msg->pkey; - primary_path->sl = cm_req_get_primary_sl(req_msg); + primary_path->pkey = + cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); + primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg); primary_path->mtu_selector = IB_SA_EQ; - primary_path->mtu = cm_req_get_path_mtu(req_msg); + primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); primary_path->rate_selector = IB_SA_EQ; - primary_path->rate = cm_req_get_primary_packet_rate(req_msg); + primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg); primary_path->packet_life_time_selector = IB_SA_EQ; primary_path->packet_life_time = - cm_req_get_primary_local_ack_timeout(req_msg); + IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg); primary_path->packet_life_time -= (primary_path->packet_life_time > 0); - primary_path->service_id = req_msg->service_id; + primary_path->service_id = + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); if (sa_path_is_roce(primary_path)) primary_path->roce.route_resolved = false; if (cm_req_has_alt_path(req_msg)) { - alt_path->dgid = req_msg->alt_local_gid; - alt_path->sgid = req_msg->alt_remote_gid; - alt_path->flow_label = cm_req_get_alt_flow_label(req_msg); - alt_path->hop_limit = req_msg->alt_hop_limit; - alt_path->traffic_class = req_msg->alt_traffic_class; + alt_path->dgid = *IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg); + alt_path->sgid = *IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg); + alt_path->flow_label = cpu_to_be32( + IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg)); + alt_path->hop_limit = + IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg); + alt_path->traffic_class = + IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg); alt_path->reversible = 1; - alt_path->pkey = req_msg->pkey; - alt_path->sl = cm_req_get_alt_sl(req_msg); + alt_path->pkey = + cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); + alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg); alt_path->mtu_selector = IB_SA_EQ; - alt_path->mtu = cm_req_get_path_mtu(req_msg); + alt_path->mtu = + IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); alt_path->rate_selector = IB_SA_EQ; - alt_path->rate = cm_req_get_alt_packet_rate(req_msg); + alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg); alt_path->packet_life_time_selector = IB_SA_EQ; alt_path->packet_life_time = - cm_req_get_alt_local_ack_timeout(req_msg); + IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg); alt_path->packet_life_time -= (alt_path->packet_life_time > 0); - alt_path->service_id = req_msg->service_id; + alt_path->service_id = + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); if (sa_path_is_roce(alt_path)) alt_path->roce.route_resolved = false; @@ -1664,23 +1716,25 @@ static void cm_format_req_event(struct cm_work *work, } else { param->alternate_path = NULL; } - param->remote_ca_guid = req_msg->local_ca_guid; - param->remote_qkey = be32_to_cpu(req_msg->local_qkey); - param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg)); + param->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); + param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg); + param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg); param->qp_type = cm_req_get_qp_type(req_msg); - param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg)); - param->responder_resources = cm_req_get_init_depth(req_msg); - param->initiator_depth = cm_req_get_resp_res(req_msg); + param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg); + param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); + param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); param->local_cm_response_timeout = - cm_req_get_remote_resp_timeout(req_msg); - param->flow_control = cm_req_get_flow_ctrl(req_msg); + IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg); + param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg); param->remote_cm_response_timeout = - cm_req_get_local_resp_timeout(req_msg); - param->retry_count = cm_req_get_retry_count(req_msg); - param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg); - param->srq = cm_req_get_srq(req_msg); + IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg); + param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); + param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); + param->srq = IBA_GET(CM_REQ_SRQ, req_msg); param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; - work->cm_event.private_data = &req_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg); } static void cm_process_work(struct cm_id_private *cm_id_priv, @@ -1714,13 +1768,16 @@ static void cm_format_mra(struct cm_mra_msg *mra_msg, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid); - cm_mra_set_msg_mraed(mra_msg, msg_mraed); - mra_msg->local_comm_id = cm_id_priv->id.local_id; - mra_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_mra_set_service_timeout(mra_msg, service_timeout); + IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed); + IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); + IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout); if (private_data && private_data_len) - memcpy(mra_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data, + private_data_len); } static void cm_format_rej(struct cm_rej_msg *rej_msg, @@ -1732,36 +1789,42 @@ static void cm_format_rej(struct cm_rej_msg *rej_msg, u8 private_data_len) { cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid); - rej_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); switch(cm_id_priv->id.state) { case IB_CM_REQ_RCVD: - rej_msg->local_comm_id = 0; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ); + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0)); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_MRA_REQ_SENT: - rej_msg->local_comm_id = cm_id_priv->id.local_id; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ); + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: - rej_msg->local_comm_id = cm_id_priv->id.local_id; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP); + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP); break; default: - rej_msg->local_comm_id = cm_id_priv->id.local_id; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER); + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, + CM_MSG_RESPONSE_OTHER); break; } - rej_msg->reason = cpu_to_be16(reason); + IBA_SET(CM_REJ_REASON, rej_msg, reason); if (ari && ari_length) { - cm_rej_set_reject_info_len(rej_msg, ari_length); - memcpy(rej_msg->ari, ari, ari_length); + IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); + IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); } if (private_data && private_data_len) - memcpy(rej_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data, + private_data_len); } static void cm_dup_req_handler(struct cm_work *work, @@ -1821,7 +1884,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work, spin_lock_irq(&cm.lock); timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info); if (timewait_info) { - cur_cm_id_priv = cm_get_id(timewait_info->work.local_id, + cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, timewait_info->work.remote_id); spin_unlock_irq(&cm.lock); if (cur_cm_id_priv) { @@ -1835,7 +1898,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work, timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info); if (timewait_info) { cm_cleanup_timewait(cm_id_priv->timewait_info); - cur_cm_id_priv = cm_get_id(timewait_info->work.local_id, + cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, timewait_info->work.remote_id); spin_unlock_irq(&cm.lock); @@ -1851,8 +1914,9 @@ static struct cm_id_private * cm_match_req(struct cm_work *work, } /* Find matching listen request. */ - listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device, - req_msg->service_id); + listen_cm_id_priv = cm_find_listen( + cm_id_priv->id.device, + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg))); if (!listen_cm_id_priv) { cm_cleanup_timewait(cm_id_priv->timewait_info); spin_unlock_irq(&cm.lock); @@ -1877,24 +1941,32 @@ out: */ static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc) { - if (!cm_req_get_primary_subnet_local(req_msg)) { - if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) { - req_msg->primary_local_lid = ib_lid_be16(wc->slid); - cm_req_set_primary_sl(req_msg, wc->sl); + if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) { + if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) { + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(ib_lid_be16(wc->slid))); + IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl); } - if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE) - req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits); + if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) + IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, + wc->dlid_path_bits); } - if (!cm_req_get_alt_subnet_local(req_msg)) { - if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) { - req_msg->alt_local_lid = ib_lid_be16(wc->slid); - cm_req_set_alt_sl(req_msg, wc->sl); + if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) { + if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) { + IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, + be16_to_cpu(ib_lid_be16(wc->slid))); + IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl); } - if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE) - req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits); + if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) + IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, + wc->dlid_path_bits); } } @@ -1914,7 +1986,8 @@ static int cm_req_handler(struct cm_work *work) return PTR_ERR(cm_id); cm_id_priv = container_of(cm_id, struct cm_id_private, id); - cm_id_priv->id.remote_id = req_msg->local_comm_id; + cm_id_priv->id.remote_id = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg)); ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, work->mad_recv_wc->recv_buf.grh, &cm_id_priv->av); @@ -1926,9 +1999,12 @@ static int cm_req_handler(struct cm_work *work) ret = PTR_ERR(cm_id_priv->timewait_info); goto destroy; } - cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id; - cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid; - cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg); + cm_id_priv->timewait_info->work.remote_id = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg)); + cm_id_priv->timewait_info->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); + cm_id_priv->timewait_info->remote_qpn = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); listen_cm_id_priv = cm_match_req(work, cm_id_priv); if (!listen_cm_id_priv) { @@ -1940,7 +2016,8 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; cm_id_priv->id.context = listen_cm_id_priv->id.context; - cm_id_priv->id.service_id = req_msg->service_id; + cm_id_priv->id.service_id = + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_process_routed_req(req_msg, work->mad_recv_wc->wc); @@ -1957,10 +2034,11 @@ static int cm_req_handler(struct cm_work *work) work->path[0].rec_type = sa_conv_gid_to_pathrec_type(gid_attr->gid_type); } else { - cm_path_set_rec_type(work->port->cm_dev->ib_device, - work->port->port_num, - &work->path[0], - &req_msg->primary_local_gid); + cm_path_set_rec_type( + work->port->cm_dev->ib_device, work->port->port_num, + &work->path[0], + IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, + req_msg)); } if (cm_req_has_alt_path(req_msg)) work->path[1].rec_type = work->path[0].rec_type; @@ -2000,16 +2078,19 @@ static int cm_req_handler(struct cm_work *work) } cm_id_priv->tid = req_msg->hdr.tid; cm_id_priv->timeout_ms = cm_convert_to_ms( - cm_req_get_local_resp_timeout(req_msg)); - cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg); - cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg); - cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg); - cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg); - cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg); - cm_id_priv->pkey = req_msg->pkey; - cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg); - cm_id_priv->retry_count = cm_req_get_retry_count(req_msg); - cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg); + IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg)); + cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg); + cm_id_priv->remote_qpn = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); + cm_id_priv->initiator_depth = + IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); + cm_id_priv->responder_resources = + IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); + cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); + cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); + cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); + cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); + cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); cm_id_priv->qp_type = cm_req_get_qp_type(req_msg); cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id); @@ -2032,29 +2113,35 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg, struct ib_cm_rep_param *param) { cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid); - rep_msg->local_comm_id = cm_id_priv->id.local_id; - rep_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn)); - rep_msg->resp_resources = param->responder_resources; - cm_rep_set_target_ack_delay(rep_msg, - cm_id_priv->av.port->cm_dev->ack_delay); - cm_rep_set_failover(rep_msg, param->failover_accepted); - cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count); - rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid; + IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); + IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn); + IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg, + param->responder_resources); + IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg, + cm_id_priv->av.port->cm_dev->ack_delay); + IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted); + IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count); + IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg, + be64_to_cpu(cm_id_priv->id.device->node_guid)); if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) { - rep_msg->initiator_depth = param->initiator_depth; - cm_rep_set_flow_ctrl(rep_msg, param->flow_control); - cm_rep_set_srq(rep_msg, param->srq); - cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num)); + IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg, + param->initiator_depth); + IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg, + param->flow_control); + IBA_SET(CM_REP_SRQ, rep_msg, param->srq); + IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num); } else { - cm_rep_set_srq(rep_msg, 1); - cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param->qp_num)); + IBA_SET(CM_REP_SRQ, rep_msg, 1); + IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num); } if (param->private_data && param->private_data_len) - memcpy(rep_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data, + param->private_data_len); } int ib_send_cm_rep(struct ib_cm_id *cm_id, @@ -2100,7 +2187,7 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id, cm_id_priv->msg = msg; cm_id_priv->initiator_depth = param->initiator_depth; cm_id_priv->responder_resources = param->responder_resources; - cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg); + cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); @@ -2114,11 +2201,14 @@ static void cm_format_rtu(struct cm_rtu_msg *rtu_msg, u8 private_data_len) { cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid); - rtu_msg->local_comm_id = cm_id_priv->id.local_id; - rtu_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); if (private_data && private_data_len) - memcpy(rtu_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data, + private_data_len); } int ib_send_cm_rtu(struct ib_cm_id *cm_id, @@ -2181,18 +2271,20 @@ static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.rep_rcvd; - param->remote_ca_guid = rep_msg->local_ca_guid; - param->remote_qkey = be32_to_cpu(rep_msg->local_qkey); + param->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); + param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg); param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type)); - param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg)); - param->responder_resources = rep_msg->initiator_depth; - param->initiator_depth = rep_msg->resp_resources; - param->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg); - param->failover_accepted = cm_rep_get_failover(rep_msg); - param->flow_control = cm_rep_get_flow_ctrl(rep_msg); - param->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg); - param->srq = cm_rep_get_srq(rep_msg); - work->cm_event.private_data = &rep_msg->private_data; + param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg); + param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); + param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); + param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); + param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg); + param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg); + param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); + param->srq = IBA_GET(CM_REP_SRQ, rep_msg); + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg); } static void cm_dup_rep_handler(struct cm_work *work) @@ -2203,8 +2295,9 @@ static void cm_dup_rep_handler(struct cm_work *work) int ret; rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, - rep_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), + cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg))); if (!cm_id_priv) return; @@ -2248,11 +2341,12 @@ static int cm_rep_handler(struct cm_work *work) struct cm_timewait_info *timewait_info; rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0); if (!cm_id_priv) { cm_dup_rep_handler(work); pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__, - be32_to_cpu(rep_msg->remote_comm_id)); + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); return -EINVAL; } @@ -2266,15 +2360,18 @@ static int cm_rep_handler(struct cm_work *work) default: spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; - pr_debug("%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", - __func__, cm_id_priv->id.state, - be32_to_cpu(rep_msg->local_comm_id), - be32_to_cpu(rep_msg->remote_comm_id)); + pr_debug( + "%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", + __func__, cm_id_priv->id.state, + IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); goto error; } - cm_id_priv->timewait_info->work.remote_id = rep_msg->local_comm_id; - cm_id_priv->timewait_info->remote_ca_guid = rep_msg->local_ca_guid; + cm_id_priv->timewait_info->work.remote_id = + cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); + cm_id_priv->timewait_info->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); spin_lock(&cm.lock); @@ -2284,7 +2381,7 @@ static int cm_rep_handler(struct cm_work *work) spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; pr_debug("%s: Failed to insert remote id %d\n", __func__, - be32_to_cpu(rep_msg->remote_comm_id)); + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); goto error; } /* Check for a stale connection. */ @@ -2293,7 +2390,7 @@ static int cm_rep_handler(struct cm_work *work) rb_erase(&cm_id_priv->timewait_info->remote_id_node, &cm.remote_id_table); cm_id_priv->timewait_info->inserted_remote_id = 0; - cur_cm_id_priv = cm_get_id(timewait_info->work.local_id, + cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, timewait_info->work.remote_id); spin_unlock(&cm.lock); @@ -2302,9 +2399,10 @@ static int cm_rep_handler(struct cm_work *work) IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP, NULL, 0); ret = -EINVAL; - pr_debug("%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", - __func__, be32_to_cpu(rep_msg->local_comm_id), - be32_to_cpu(rep_msg->remote_comm_id)); + pr_debug( + "%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", + __func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); if (cur_cm_id_priv) { cm_id = &cur_cm_id_priv->id; @@ -2317,13 +2415,17 @@ static int cm_rep_handler(struct cm_work *work) spin_unlock(&cm.lock); cm_id_priv->id.state = IB_CM_REP_RCVD; - cm_id_priv->id.remote_id = rep_msg->local_comm_id; + cm_id_priv->id.remote_id = + cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); - cm_id_priv->initiator_depth = rep_msg->resp_resources; - cm_id_priv->responder_resources = rep_msg->initiator_depth; - cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg); - cm_id_priv->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg); - cm_id_priv->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg); + cm_id_priv->initiator_depth = + IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); + cm_id_priv->responder_resources = + IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); + cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); + cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); + cm_id_priv->target_ack_delay = + IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); cm_id_priv->av.timeout = cm_ack_timeout(cm_id_priv->target_ack_delay, cm_id_priv->av.timeout - 1); @@ -2389,12 +2491,14 @@ static int cm_rtu_handler(struct cm_work *work) int ret; rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(rtu_msg->remote_comm_id, - rtu_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)), + cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg))); if (!cm_id_priv) return -EINVAL; - work->cm_event.private_data = &rtu_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_REP_SENT && @@ -2429,12 +2533,16 @@ static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, { cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID, cm_form_tid(cm_id_priv)); - dreq_msg->local_comm_id = cm_id_priv->id.local_id; - dreq_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv->remote_qpn); + IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); + IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg, + be32_to_cpu(cm_id_priv->remote_qpn)); if (private_data && private_data_len) - memcpy(dreq_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data, + private_data_len); } int ib_send_cm_dreq(struct ib_cm_id *cm_id, @@ -2494,11 +2602,14 @@ static void cm_format_drep(struct cm_drep_msg *drep_msg, u8 private_data_len) { cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid); - drep_msg->local_comm_id = cm_id_priv->id.local_id; - drep_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); if (private_data && private_data_len) - memcpy(drep_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data, + private_data_len); } int ib_send_cm_drep(struct ib_cm_id *cm_id, @@ -2566,8 +2677,10 @@ static int cm_issue_drep(struct cm_port *port, drep_msg = (struct cm_drep_msg *) msg->mad; cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid); - drep_msg->remote_comm_id = dreq_msg->local_comm_id; - drep_msg->local_comm_id = dreq_msg->remote_comm_id; + IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, + IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)); + IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, + IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); ret = ib_post_send_mad(msg, NULL); if (ret) @@ -2584,22 +2697,26 @@ static int cm_dreq_handler(struct cm_work *work) int ret; dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id, - dreq_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)), + cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg))); if (!cm_id_priv) { atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_DREQ_COUNTER]); cm_issue_drep(work->port, work->mad_recv_wc); - pr_debug("%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", - __func__, be32_to_cpu(dreq_msg->local_comm_id), - be32_to_cpu(dreq_msg->remote_comm_id)); + pr_debug( + "%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", + __func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg), + IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); return -EINVAL; } - work->cm_event.private_data = &dreq_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg); spin_lock_irq(&cm_id_priv->lock); - if (cm_id_priv->local_qpn != cm_dreq_get_remote_qpn(dreq_msg)) + if (cm_id_priv->local_qpn != + cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg))) goto unlock; switch (cm_id_priv->id.state) { @@ -2665,12 +2782,14 @@ static int cm_drep_handler(struct cm_work *work) int ret; drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(drep_msg->remote_comm_id, - drep_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)), + cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg))); if (!cm_id_priv) return -EINVAL; - work->cm_event.private_data = &drep_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_DREQ_SENT && @@ -2766,10 +2885,11 @@ static void cm_format_rej_event(struct cm_work *work) rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.rej_rcvd; - param->ari = rej_msg->ari; - param->ari_length = cm_rej_get_reject_info_len(rej_msg); - param->reason = __be16_to_cpu(rej_msg->reason); - work->cm_event.private_data = &rej_msg->private_data; + param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg); + param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg); + param->reason = IBA_GET(CM_REJ_REASON, rej_msg); + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg); } static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) @@ -2778,29 +2898,29 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) struct cm_id_private *cm_id_priv; __be32 remote_id; - remote_id = rej_msg->local_comm_id; + remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg)); - if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_TIMEOUT) { + if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) { spin_lock_irq(&cm.lock); - timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari), - remote_id); + timewait_info = cm_find_remote_id( + *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)), + remote_id); if (!timewait_info) { spin_unlock_irq(&cm.lock); return NULL; } - cm_id_priv = xa_load(&cm.local_id_table, - cm_local_id(timewait_info->work.local_id)); - if (cm_id_priv) { - if (cm_id_priv->id.remote_id == remote_id) - refcount_inc(&cm_id_priv->refcount); - else - cm_id_priv = NULL; - } + cm_id_priv = + cm_acquire_id(timewait_info->work.local_id, remote_id); spin_unlock_irq(&cm.lock); - } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ) - cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0); + } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) == + CM_MSG_RESPONSE_REQ) + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), + 0); else - cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), + remote_id); return cm_id_priv; } @@ -2828,7 +2948,7 @@ static int cm_rej_handler(struct cm_work *work) /* fall through */ case IB_CM_REQ_RCVD: case IB_CM_MRA_REQ_SENT: - if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_STALE_CONN) + if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN) cm_enter_timewait(cm_id_priv); else cm_reset_to_idle(cm_id_priv); @@ -2958,13 +3078,16 @@ EXPORT_SYMBOL(ib_send_cm_mra); static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) { - switch (cm_mra_get_msg_mraed(mra_msg)) { + switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) { case CM_MSG_RESPONSE_REQ: - return cm_acquire_id(mra_msg->remote_comm_id, 0); + return cm_acquire_id( + cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), + 0); case CM_MSG_RESPONSE_REP: case CM_MSG_RESPONSE_OTHER: - return cm_acquire_id(mra_msg->remote_comm_id, - mra_msg->local_comm_id); + return cm_acquire_id( + cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), + cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg))); default: return NULL; } @@ -2981,30 +3104,34 @@ static int cm_mra_handler(struct cm_work *work) if (!cm_id_priv) return -EINVAL; - work->cm_event.private_data = &mra_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg); work->cm_event.param.mra_rcvd.service_timeout = - cm_mra_get_service_timeout(mra_msg); - timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) + + IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg); + timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) + cm_convert_to_ms(cm_id_priv->av.timeout); spin_lock_irq(&cm_id_priv->lock); switch (cm_id_priv->id.state) { case IB_CM_REQ_SENT: - if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ || + if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != + CM_MSG_RESPONSE_REQ || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) goto out; cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD; break; case IB_CM_REP_SENT: - if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP || + if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != + CM_MSG_RESPONSE_REP || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) goto out; cm_id_priv->id.state = IB_CM_MRA_REP_RCVD; break; case IB_CM_ESTABLISHED: - if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER || + if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != + CM_MSG_RESPONSE_OTHER || cm_id_priv->id.lap_state != IB_CM_LAP_SENT || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) { @@ -3046,117 +3173,23 @@ out: return -EINVAL; } -static void cm_format_lap(struct cm_lap_msg *lap_msg, - struct cm_id_private *cm_id_priv, - struct sa_path_rec *alternate_path, - const void *private_data, - u8 private_data_len) -{ - bool alt_ext = false; - - if (alternate_path->rec_type == SA_PATH_REC_TYPE_OPA) - alt_ext = opa_is_extended_lid(alternate_path->opa.dlid, - alternate_path->opa.slid); - cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID, - cm_form_tid(cm_id_priv)); - lap_msg->local_comm_id = cm_id_priv->id.local_id; - lap_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_lap_set_remote_qpn(lap_msg, cm_id_priv->remote_qpn); - /* todo: need remote CM response timeout */ - cm_lap_set_remote_resp_timeout(lap_msg, 0x1F); - lap_msg->alt_local_lid = - htons(ntohl(sa_path_get_slid(alternate_path))); - lap_msg->alt_remote_lid = - htons(ntohl(sa_path_get_dlid(alternate_path))); - lap_msg->alt_local_gid = alternate_path->sgid; - lap_msg->alt_remote_gid = alternate_path->dgid; - if (alt_ext) { - lap_msg->alt_local_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.slid)); - lap_msg->alt_remote_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.dlid)); - } - cm_lap_set_flow_label(lap_msg, alternate_path->flow_label); - cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class); - lap_msg->alt_hop_limit = alternate_path->hop_limit; - cm_lap_set_packet_rate(lap_msg, alternate_path->rate); - cm_lap_set_sl(lap_msg, alternate_path->sl); - cm_lap_set_subnet_local(lap_msg, 1); /* local only... */ - cm_lap_set_local_ack_timeout(lap_msg, - cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, - alternate_path->packet_life_time)); - - if (private_data && private_data_len) - memcpy(lap_msg->private_data, private_data, private_data_len); -} - -int ib_send_cm_lap(struct ib_cm_id *cm_id, - struct sa_path_rec *alternate_path, - const void *private_data, - u8 private_data_len) -{ - struct cm_id_private *cm_id_priv; - struct ib_mad_send_buf *msg; - unsigned long flags; - int ret; - - if (private_data && private_data_len > IB_CM_LAP_PRIVATE_DATA_SIZE) - return -EINVAL; - - cm_id_priv = container_of(cm_id, struct cm_id_private, id); - spin_lock_irqsave(&cm_id_priv->lock, flags); - if (cm_id->state != IB_CM_ESTABLISHED || - (cm_id->lap_state != IB_CM_LAP_UNINIT && - cm_id->lap_state != IB_CM_LAP_IDLE)) { - ret = -EINVAL; - goto out; - } - - ret = cm_init_av_by_path(alternate_path, NULL, &cm_id_priv->alt_av, - cm_id_priv); - if (ret) - goto out; - cm_id_priv->alt_av.timeout = - cm_ack_timeout(cm_id_priv->target_ack_delay, - cm_id_priv->alt_av.timeout - 1); - - ret = cm_alloc_msg(cm_id_priv, &msg); - if (ret) - goto out; - - cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv, - alternate_path, private_data, private_data_len); - msg->timeout_ms = cm_id_priv->timeout_ms; - msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED; - - ret = ib_post_send_mad(msg, NULL); - if (ret) { - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - cm_free_msg(msg); - return ret; - } - - cm_id->lap_state = IB_CM_LAP_SENT; - cm_id_priv->msg = msg; - -out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); - return ret; -} -EXPORT_SYMBOL(ib_send_cm_lap); - static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg, struct sa_path_rec *path) { u32 lid; if (path->rec_type != SA_PATH_REC_TYPE_OPA) { - sa_path_set_dlid(path, ntohs(lap_msg->alt_local_lid)); - sa_path_set_slid(path, ntohs(lap_msg->alt_remote_lid)); + sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID, + lap_msg)); + sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID, + lap_msg)); } else { - lid = opa_get_lid_from_gid(&lap_msg->alt_local_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg)); sa_path_set_dlid(path, lid); - lid = opa_get_lid_from_gid(&lap_msg->alt_remote_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg)); sa_path_set_slid(path, lid); } } @@ -3165,20 +3198,23 @@ static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, struct sa_path_rec *path, struct cm_lap_msg *lap_msg) { - path->dgid = lap_msg->alt_local_gid; - path->sgid = lap_msg->alt_remote_gid; - path->flow_label = cm_lap_get_flow_label(lap_msg); - path->hop_limit = lap_msg->alt_hop_limit; - path->traffic_class = cm_lap_get_traffic_class(lap_msg); + path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg); + path->sgid = + *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg); + path->flow_label = + cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg)); + path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg); + path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg); path->reversible = 1; path->pkey = cm_id_priv->pkey; - path->sl = cm_lap_get_sl(lap_msg); + path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg); path->mtu_selector = IB_SA_EQ; path->mtu = cm_id_priv->path_mtu; path->rate_selector = IB_SA_EQ; - path->rate = cm_lap_get_packet_rate(lap_msg); + path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg); path->packet_life_time_selector = IB_SA_EQ; - path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg); + path->packet_life_time = + IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg); path->packet_life_time -= (path->packet_life_time > 0); cm_format_path_lid_from_lap(lap_msg, path); } @@ -3200,20 +3236,22 @@ static int cm_lap_handler(struct cm_work *work) /* todo: verify LAP request and send reject APR if invalid. */ lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(lap_msg->remote_comm_id, - lap_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)), + cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg))); if (!cm_id_priv) return -EINVAL; param = &work->cm_event.param.lap_rcvd; memset(&work->path[0], 0, sizeof(work->path[1])); cm_path_set_rec_type(work->port->cm_dev->ib_device, - work->port->port_num, - &work->path[0], - &lap_msg->alt_local_gid); + work->port->port_num, &work->path[0], + IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, + lap_msg)); param->alternate_path = &work->path[0]; cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg); - work->cm_event.private_data = &lap_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_ESTABLISHED) @@ -3278,72 +3316,6 @@ deref: cm_deref_id(cm_id_priv); return -EINVAL; } -static void cm_format_apr(struct cm_apr_msg *apr_msg, - struct cm_id_private *cm_id_priv, - enum ib_cm_apr_status status, - void *info, - u8 info_length, - const void *private_data, - u8 private_data_len) -{ - cm_format_mad_hdr(&apr_msg->hdr, CM_APR_ATTR_ID, cm_id_priv->tid); - apr_msg->local_comm_id = cm_id_priv->id.local_id; - apr_msg->remote_comm_id = cm_id_priv->id.remote_id; - apr_msg->ap_status = (u8) status; - - if (info && info_length) { - apr_msg->info_length = info_length; - memcpy(apr_msg->info, info, info_length); - } - - if (private_data && private_data_len) - memcpy(apr_msg->private_data, private_data, private_data_len); -} - -int ib_send_cm_apr(struct ib_cm_id *cm_id, - enum ib_cm_apr_status status, - void *info, - u8 info_length, - const void *private_data, - u8 private_data_len) -{ - struct cm_id_private *cm_id_priv; - struct ib_mad_send_buf *msg; - unsigned long flags; - int ret; - - if ((private_data && private_data_len > IB_CM_APR_PRIVATE_DATA_SIZE) || - (info && info_length > IB_CM_APR_INFO_LENGTH)) - return -EINVAL; - - cm_id_priv = container_of(cm_id, struct cm_id_private, id); - spin_lock_irqsave(&cm_id_priv->lock, flags); - if (cm_id->state != IB_CM_ESTABLISHED || - (cm_id->lap_state != IB_CM_LAP_RCVD && - cm_id->lap_state != IB_CM_MRA_LAP_SENT)) { - ret = -EINVAL; - goto out; - } - - ret = cm_alloc_msg(cm_id_priv, &msg); - if (ret) - goto out; - - cm_format_apr((struct cm_apr_msg *) msg->mad, cm_id_priv, status, - info, info_length, private_data, private_data_len); - ret = ib_post_send_mad(msg, NULL); - if (ret) { - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - cm_free_msg(msg); - return ret; - } - - cm_id->lap_state = IB_CM_LAP_IDLE; -out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); - return ret; -} -EXPORT_SYMBOL(ib_send_cm_apr); - static int cm_apr_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; @@ -3358,15 +3330,20 @@ static int cm_apr_handler(struct cm_work *work) return -EINVAL; apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(apr_msg->remote_comm_id, - apr_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)), + cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg))); if (!cm_id_priv) return -EINVAL; /* Unmatched reply. */ - work->cm_event.param.apr_rcvd.ap_status = apr_msg->ap_status; - work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info; - work->cm_event.param.apr_rcvd.info_len = apr_msg->info_length; - work->cm_event.private_data = &apr_msg->private_data; + work->cm_event.param.apr_rcvd.ap_status = + IBA_GET(CM_APR_AR_STATUS, apr_msg); + work->cm_event.param.apr_rcvd.apr_info = + IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg); + work->cm_event.param.apr_rcvd.info_len = + IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg); + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_ESTABLISHED || @@ -3438,13 +3415,16 @@ static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg, { cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID, cm_form_tid(cm_id_priv)); - sidr_req_msg->request_id = cm_id_priv->id.local_id; - sidr_req_msg->pkey = param->path->pkey; - sidr_req_msg->service_id = param->service_id; + IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg, + be16_to_cpu(param->path->pkey)); + IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg, + be64_to_cpu(param->service_id)); if (param->private_data && param->private_data_len) - memcpy(sidr_req_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg, + param->private_data, param->private_data_len); } int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, @@ -3508,13 +3488,15 @@ static void cm_format_sidr_req_event(struct cm_work *work, sidr_req_msg = (struct cm_sidr_req_msg *) work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.sidr_req_rcvd; - param->pkey = __be16_to_cpu(sidr_req_msg->pkey); + param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg); param->listen_id = listen_id; - param->service_id = sidr_req_msg->service_id; + param->service_id = + cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); param->bth_pkey = cm_get_bth_pkey(work); param->port = work->port->port_num; param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr; - work->cm_event.private_data = &sidr_req_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg); } static int cm_sidr_req_handler(struct cm_work *work) @@ -3542,7 +3524,8 @@ static int cm_sidr_req_handler(struct cm_work *work) if (ret) goto out; - cm_id_priv->id.remote_id = sidr_req_msg->request_id; + cm_id_priv->id.remote_id = + cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg)); cm_id_priv->tid = sidr_req_msg->hdr.tid; atomic_inc(&cm_id_priv->work_count); @@ -3555,8 +3538,9 @@ static int cm_sidr_req_handler(struct cm_work *work) goto out; /* Duplicate message. */ } cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD; - cur_cm_id_priv = cm_find_listen(cm_id->device, - sidr_req_msg->service_id); + cur_cm_id_priv = cm_find_listen( + cm_id->device, + cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg))); if (!cur_cm_id_priv) { spin_unlock_irq(&cm.lock); cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED); @@ -3568,7 +3552,8 @@ static int cm_sidr_req_handler(struct cm_work *work) cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler; cm_id_priv->id.context = cur_cm_id_priv->id.context; - cm_id_priv->id.service_id = sidr_req_msg->service_id; + cm_id_priv->id.service_id = + cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_format_sidr_req_event(work, cm_id_priv, &cur_cm_id_priv->id); @@ -3586,18 +3571,21 @@ static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, { cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID, cm_id_priv->tid); - sidr_rep_msg->request_id = cm_id_priv->id.remote_id; - sidr_rep_msg->status = param->status; - cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param->qp_num)); - sidr_rep_msg->service_id = cm_id_priv->id.service_id; - sidr_rep_msg->qkey = cpu_to_be32(param->qkey); + IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); + IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status); + IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num); + IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg, + be64_to_cpu(cm_id_priv->id.service_id)); + IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey); if (param->info && param->info_length) - memcpy(sidr_rep_msg->info, param->info, param->info_length); + IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg, + param->info, param->info_length); if (param->private_data && param->private_data_len) - memcpy(sidr_rep_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg, + param->private_data, param->private_data_len); } int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id, @@ -3657,13 +3645,16 @@ static void cm_format_sidr_rep_event(struct cm_work *work, sidr_rep_msg = (struct cm_sidr_rep_msg *) work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.sidr_rep_rcvd; - param->status = sidr_rep_msg->status; - param->qkey = be32_to_cpu(sidr_rep_msg->qkey); - param->qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg)); - param->info = &sidr_rep_msg->info; - param->info_len = sidr_rep_msg->info_length; + param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg); + param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg); + param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg); + param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION, + sidr_rep_msg); + param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH, + sidr_rep_msg); param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; - work->cm_event.private_data = &sidr_rep_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg); } static int cm_sidr_rep_handler(struct cm_work *work) @@ -3673,7 +3664,8 @@ static int cm_sidr_rep_handler(struct cm_work *work) sidr_rep_msg = (struct cm_sidr_rep_msg *) work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(sidr_rep_msg->request_id, 0); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0); if (!cm_id_priv) return -EINVAL; /* Unmatched reply. */ diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h index 92d7260ac913..0cc40656b5c5 100644 --- a/drivers/infiniband/core/cm_msgs.h +++ b/drivers/infiniband/core/cm_msgs.h @@ -8,6 +8,7 @@ #ifndef CM_MSGS_H #define CM_MSGS_H +#include <rdma/ibta_vol1_c12.h> #include <rdma/ib_mad.h> #include <rdma/ib_cm.h> @@ -18,120 +19,14 @@ #define IB_CM_CLASS_VERSION 2 /* IB specification 1.2 */ -struct cm_req_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 rsvd4; - __be64 service_id; - __be64 local_ca_guid; - __be32 rsvd24; - __be32 local_qkey; - /* local QPN:24, responder resources:8 */ - __be32 offset32; - /* local EECN:24, initiator depth:8 */ - __be32 offset36; - /* - * remote EECN:24, remote CM response timeout:5, - * transport service type:2, end-to-end flow control:1 - */ - __be32 offset40; - /* starting PSN:24, local CM response timeout:5, retry count:3 */ - __be32 offset44; - __be16 pkey; - /* path MTU:4, RDC exists:1, RNR retry count:3. */ - u8 offset50; - /* max CM Retries:4, SRQ:1, extended transport type:3 */ - u8 offset51; - - __be16 primary_local_lid; - __be16 primary_remote_lid; - union ib_gid primary_local_gid; - union ib_gid primary_remote_gid; - /* flow label:20, rsvd:6, packet rate:6 */ - __be32 primary_offset88; - u8 primary_traffic_class; - u8 primary_hop_limit; - /* SL:4, subnet local:1, rsvd:3 */ - u8 primary_offset94; - /* local ACK timeout:5, rsvd:3 */ - u8 primary_offset95; - - __be16 alt_local_lid; - __be16 alt_remote_lid; - union ib_gid alt_local_gid; - union ib_gid alt_remote_gid; - /* flow label:20, rsvd:6, packet rate:6 */ - __be32 alt_offset132; - u8 alt_traffic_class; - u8 alt_hop_limit; - /* SL:4, subnet local:1, rsvd:3 */ - u8 alt_offset138; - /* local ACK timeout:5, rsvd:3 */ - u8 alt_offset139; - - u32 private_data[IB_CM_REQ_PRIVATE_DATA_SIZE / sizeof(u32)]; - -} __packed; - -static inline __be32 cm_req_get_local_qpn(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->offset32) >> 8); -} - -static inline void cm_req_set_local_qpn(struct cm_req_msg *req_msg, __be32 qpn) -{ - req_msg->offset32 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(req_msg->offset32) & - 0x000000FF)); -} - -static inline u8 cm_req_get_resp_res(struct cm_req_msg *req_msg) -{ - return (u8) be32_to_cpu(req_msg->offset32); -} - -static inline void cm_req_set_resp_res(struct cm_req_msg *req_msg, u8 resp_res) -{ - req_msg->offset32 = cpu_to_be32(resp_res | - (be32_to_cpu(req_msg->offset32) & - 0xFFFFFF00)); -} - -static inline u8 cm_req_get_init_depth(struct cm_req_msg *req_msg) -{ - return (u8) be32_to_cpu(req_msg->offset36); -} - -static inline void cm_req_set_init_depth(struct cm_req_msg *req_msg, - u8 init_depth) -{ - req_msg->offset36 = cpu_to_be32(init_depth | - (be32_to_cpu(req_msg->offset36) & - 0xFFFFFF00)); -} - -static inline u8 cm_req_get_remote_resp_timeout(struct cm_req_msg *req_msg) -{ - return (u8) ((be32_to_cpu(req_msg->offset40) & 0xF8) >> 3); -} - -static inline void cm_req_set_remote_resp_timeout(struct cm_req_msg *req_msg, - u8 resp_timeout) -{ - req_msg->offset40 = cpu_to_be32((resp_timeout << 3) | - (be32_to_cpu(req_msg->offset40) & - 0xFFFFFF07)); -} - static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg) { - u8 transport_type = (u8) (be32_to_cpu(req_msg->offset40) & 0x06) >> 1; + u8 transport_type = IBA_GET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg); switch(transport_type) { case 0: return IB_QPT_RC; case 1: return IB_QPT_UC; case 3: - switch (req_msg->offset51 & 0x7) { + switch (IBA_GET(CM_REQ_EXTENDED_TRANSPORT_TYPE, req_msg)) { case 1: return IB_QPT_XRC_TGT; default: return 0; } @@ -144,240 +39,17 @@ static inline void cm_req_set_qp_type(struct cm_req_msg *req_msg, { switch(qp_type) { case IB_QPT_UC: - req_msg->offset40 = cpu_to_be32((be32_to_cpu( - req_msg->offset40) & - 0xFFFFFFF9) | 0x2); + IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 1); break; case IB_QPT_XRC_INI: - req_msg->offset40 = cpu_to_be32((be32_to_cpu( - req_msg->offset40) & - 0xFFFFFFF9) | 0x6); - req_msg->offset51 = (req_msg->offset51 & 0xF8) | 1; + IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 3); + IBA_SET(CM_REQ_EXTENDED_TRANSPORT_TYPE, req_msg, 1); break; default: - req_msg->offset40 = cpu_to_be32(be32_to_cpu( - req_msg->offset40) & - 0xFFFFFFF9); + IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 0); } } -static inline u8 cm_req_get_flow_ctrl(struct cm_req_msg *req_msg) -{ - return be32_to_cpu(req_msg->offset40) & 0x1; -} - -static inline void cm_req_set_flow_ctrl(struct cm_req_msg *req_msg, - u8 flow_ctrl) -{ - req_msg->offset40 = cpu_to_be32((flow_ctrl & 0x1) | - (be32_to_cpu(req_msg->offset40) & - 0xFFFFFFFE)); -} - -static inline __be32 cm_req_get_starting_psn(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->offset44) >> 8); -} - -static inline void cm_req_set_starting_psn(struct cm_req_msg *req_msg, - __be32 starting_psn) -{ - req_msg->offset44 = cpu_to_be32((be32_to_cpu(starting_psn) << 8) | - (be32_to_cpu(req_msg->offset44) & 0x000000FF)); -} - -static inline u8 cm_req_get_local_resp_timeout(struct cm_req_msg *req_msg) -{ - return (u8) ((be32_to_cpu(req_msg->offset44) & 0xF8) >> 3); -} - -static inline void cm_req_set_local_resp_timeout(struct cm_req_msg *req_msg, - u8 resp_timeout) -{ - req_msg->offset44 = cpu_to_be32((resp_timeout << 3) | - (be32_to_cpu(req_msg->offset44) & 0xFFFFFF07)); -} - -static inline u8 cm_req_get_retry_count(struct cm_req_msg *req_msg) -{ - return (u8) (be32_to_cpu(req_msg->offset44) & 0x7); -} - -static inline void cm_req_set_retry_count(struct cm_req_msg *req_msg, - u8 retry_count) -{ - req_msg->offset44 = cpu_to_be32((retry_count & 0x7) | - (be32_to_cpu(req_msg->offset44) & 0xFFFFFFF8)); -} - -static inline u8 cm_req_get_path_mtu(struct cm_req_msg *req_msg) -{ - return req_msg->offset50 >> 4; -} - -static inline void cm_req_set_path_mtu(struct cm_req_msg *req_msg, u8 path_mtu) -{ - req_msg->offset50 = (u8) ((req_msg->offset50 & 0xF) | (path_mtu << 4)); -} - -static inline u8 cm_req_get_rnr_retry_count(struct cm_req_msg *req_msg) -{ - return req_msg->offset50 & 0x7; -} - -static inline void cm_req_set_rnr_retry_count(struct cm_req_msg *req_msg, - u8 rnr_retry_count) -{ - req_msg->offset50 = (u8) ((req_msg->offset50 & 0xF8) | - (rnr_retry_count & 0x7)); -} - -static inline u8 cm_req_get_max_cm_retries(struct cm_req_msg *req_msg) -{ - return req_msg->offset51 >> 4; -} - -static inline void cm_req_set_max_cm_retries(struct cm_req_msg *req_msg, - u8 retries) -{ - req_msg->offset51 = (u8) ((req_msg->offset51 & 0xF) | (retries << 4)); -} - -static inline u8 cm_req_get_srq(struct cm_req_msg *req_msg) -{ - return (req_msg->offset51 & 0x8) >> 3; -} - -static inline void cm_req_set_srq(struct cm_req_msg *req_msg, u8 srq) -{ - req_msg->offset51 = (u8) ((req_msg->offset51 & 0xF7) | - ((srq & 0x1) << 3)); -} - -static inline __be32 cm_req_get_primary_flow_label(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->primary_offset88) >> 12); -} - -static inline void cm_req_set_primary_flow_label(struct cm_req_msg *req_msg, - __be32 flow_label) -{ - req_msg->primary_offset88 = cpu_to_be32( - (be32_to_cpu(req_msg->primary_offset88) & - 0x00000FFF) | - (be32_to_cpu(flow_label) << 12)); -} - -static inline u8 cm_req_get_primary_packet_rate(struct cm_req_msg *req_msg) -{ - return (u8) (be32_to_cpu(req_msg->primary_offset88) & 0x3F); -} - -static inline void cm_req_set_primary_packet_rate(struct cm_req_msg *req_msg, - u8 rate) -{ - req_msg->primary_offset88 = cpu_to_be32( - (be32_to_cpu(req_msg->primary_offset88) & - 0xFFFFFFC0) | (rate & 0x3F)); -} - -static inline u8 cm_req_get_primary_sl(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->primary_offset94 >> 4); -} - -static inline void cm_req_set_primary_sl(struct cm_req_msg *req_msg, u8 sl) -{ - req_msg->primary_offset94 = (u8) ((req_msg->primary_offset94 & 0x0F) | - (sl << 4)); -} - -static inline u8 cm_req_get_primary_subnet_local(struct cm_req_msg *req_msg) -{ - return (u8) ((req_msg->primary_offset94 & 0x08) >> 3); -} - -static inline void cm_req_set_primary_subnet_local(struct cm_req_msg *req_msg, - u8 subnet_local) -{ - req_msg->primary_offset94 = (u8) ((req_msg->primary_offset94 & 0xF7) | - ((subnet_local & 0x1) << 3)); -} - -static inline u8 cm_req_get_primary_local_ack_timeout(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->primary_offset95 >> 3); -} - -static inline void cm_req_set_primary_local_ack_timeout(struct cm_req_msg *req_msg, - u8 local_ack_timeout) -{ - req_msg->primary_offset95 = (u8) ((req_msg->primary_offset95 & 0x07) | - (local_ack_timeout << 3)); -} - -static inline __be32 cm_req_get_alt_flow_label(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->alt_offset132) >> 12); -} - -static inline void cm_req_set_alt_flow_label(struct cm_req_msg *req_msg, - __be32 flow_label) -{ - req_msg->alt_offset132 = cpu_to_be32( - (be32_to_cpu(req_msg->alt_offset132) & - 0x00000FFF) | - (be32_to_cpu(flow_label) << 12)); -} - -static inline u8 cm_req_get_alt_packet_rate(struct cm_req_msg *req_msg) -{ - return (u8) (be32_to_cpu(req_msg->alt_offset132) & 0x3F); -} - -static inline void cm_req_set_alt_packet_rate(struct cm_req_msg *req_msg, - u8 rate) -{ - req_msg->alt_offset132 = cpu_to_be32( - (be32_to_cpu(req_msg->alt_offset132) & - 0xFFFFFFC0) | (rate & 0x3F)); -} - -static inline u8 cm_req_get_alt_sl(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->alt_offset138 >> 4); -} - -static inline void cm_req_set_alt_sl(struct cm_req_msg *req_msg, u8 sl) -{ - req_msg->alt_offset138 = (u8) ((req_msg->alt_offset138 & 0x0F) | - (sl << 4)); -} - -static inline u8 cm_req_get_alt_subnet_local(struct cm_req_msg *req_msg) -{ - return (u8) ((req_msg->alt_offset138 & 0x08) >> 3); -} - -static inline void cm_req_set_alt_subnet_local(struct cm_req_msg *req_msg, - u8 subnet_local) -{ - req_msg->alt_offset138 = (u8) ((req_msg->alt_offset138 & 0xF7) | - ((subnet_local & 0x1) << 3)); -} - -static inline u8 cm_req_get_alt_local_ack_timeout(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->alt_offset139 >> 3); -} - -static inline void cm_req_set_alt_local_ack_timeout(struct cm_req_msg *req_msg, - u8 local_ack_timeout) -{ - req_msg->alt_offset139 = (u8) ((req_msg->alt_offset139 & 0x07) | - (local_ack_timeout << 3)); -} - /* Message REJected or MRAed */ enum cm_msg_response { CM_MSG_RESPONSE_REQ = 0x0, @@ -385,419 +57,12 @@ enum cm_msg_response { CM_MSG_RESPONSE_OTHER = 0x2 }; - struct cm_mra_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - /* message MRAed:2, rsvd:6 */ - u8 offset8; - /* service timeout:5, rsvd:3 */ - u8 offset9; - - u8 private_data[IB_CM_MRA_PRIVATE_DATA_SIZE]; - -} __packed; - -static inline u8 cm_mra_get_msg_mraed(struct cm_mra_msg *mra_msg) -{ - return (u8) (mra_msg->offset8 >> 6); -} - -static inline void cm_mra_set_msg_mraed(struct cm_mra_msg *mra_msg, u8 msg) -{ - mra_msg->offset8 = (u8) ((mra_msg->offset8 & 0x3F) | (msg << 6)); -} - -static inline u8 cm_mra_get_service_timeout(struct cm_mra_msg *mra_msg) -{ - return (u8) (mra_msg->offset9 >> 3); -} - -static inline void cm_mra_set_service_timeout(struct cm_mra_msg *mra_msg, - u8 service_timeout) -{ - mra_msg->offset9 = (u8) ((mra_msg->offset9 & 0x07) | - (service_timeout << 3)); -} - -struct cm_rej_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - /* message REJected:2, rsvd:6 */ - u8 offset8; - /* reject info length:7, rsvd:1. */ - u8 offset9; - __be16 reason; - u8 ari[IB_CM_REJ_ARI_LENGTH]; - - u8 private_data[IB_CM_REJ_PRIVATE_DATA_SIZE]; - -} __packed; - -static inline u8 cm_rej_get_msg_rejected(struct cm_rej_msg *rej_msg) -{ - return (u8) (rej_msg->offset8 >> 6); -} - -static inline void cm_rej_set_msg_rejected(struct cm_rej_msg *rej_msg, u8 msg) -{ - rej_msg->offset8 = (u8) ((rej_msg->offset8 & 0x3F) | (msg << 6)); -} - -static inline u8 cm_rej_get_reject_info_len(struct cm_rej_msg *rej_msg) -{ - return (u8) (rej_msg->offset9 >> 1); -} - -static inline void cm_rej_set_reject_info_len(struct cm_rej_msg *rej_msg, - u8 len) -{ - rej_msg->offset9 = (u8) ((rej_msg->offset9 & 0x1) | (len << 1)); -} - -struct cm_rep_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - __be32 local_qkey; - /* local QPN:24, rsvd:8 */ - __be32 offset12; - /* local EECN:24, rsvd:8 */ - __be32 offset16; - /* starting PSN:24 rsvd:8 */ - __be32 offset20; - u8 resp_resources; - u8 initiator_depth; - /* target ACK delay:5, failover accepted:2, end-to-end flow control:1 */ - u8 offset26; - /* RNR retry count:3, SRQ:1, rsvd:5 */ - u8 offset27; - __be64 local_ca_guid; - - u8 private_data[IB_CM_REP_PRIVATE_DATA_SIZE]; - -} __packed; - -static inline __be32 cm_rep_get_local_qpn(struct cm_rep_msg *rep_msg) -{ - return cpu_to_be32(be32_to_cpu(rep_msg->offset12) >> 8); -} - -static inline void cm_rep_set_local_qpn(struct cm_rep_msg *rep_msg, __be32 qpn) -{ - rep_msg->offset12 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(rep_msg->offset12) & 0x000000FF)); -} - -static inline __be32 cm_rep_get_local_eecn(struct cm_rep_msg *rep_msg) -{ - return cpu_to_be32(be32_to_cpu(rep_msg->offset16) >> 8); -} - -static inline void cm_rep_set_local_eecn(struct cm_rep_msg *rep_msg, __be32 eecn) -{ - rep_msg->offset16 = cpu_to_be32((be32_to_cpu(eecn) << 8) | - (be32_to_cpu(rep_msg->offset16) & 0x000000FF)); -} - static inline __be32 cm_rep_get_qpn(struct cm_rep_msg *rep_msg, enum ib_qp_type qp_type) { return (qp_type == IB_QPT_XRC_INI) ? - cm_rep_get_local_eecn(rep_msg) : cm_rep_get_local_qpn(rep_msg); -} - -static inline __be32 cm_rep_get_starting_psn(struct cm_rep_msg *rep_msg) -{ - return cpu_to_be32(be32_to_cpu(rep_msg->offset20) >> 8); -} - -static inline void cm_rep_set_starting_psn(struct cm_rep_msg *rep_msg, - __be32 starting_psn) -{ - rep_msg->offset20 = cpu_to_be32((be32_to_cpu(starting_psn) << 8) | - (be32_to_cpu(rep_msg->offset20) & 0x000000FF)); -} - -static inline u8 cm_rep_get_target_ack_delay(struct cm_rep_msg *rep_msg) -{ - return (u8) (rep_msg->offset26 >> 3); -} - -static inline void cm_rep_set_target_ack_delay(struct cm_rep_msg *rep_msg, - u8 target_ack_delay) -{ - rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0x07) | - (target_ack_delay << 3)); -} - -static inline u8 cm_rep_get_failover(struct cm_rep_msg *rep_msg) -{ - return (u8) ((rep_msg->offset26 & 0x06) >> 1); -} - -static inline void cm_rep_set_failover(struct cm_rep_msg *rep_msg, u8 failover) -{ - rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0xF9) | - ((failover & 0x3) << 1)); -} - -static inline u8 cm_rep_get_flow_ctrl(struct cm_rep_msg *rep_msg) -{ - return (u8) (rep_msg->offset26 & 0x01); -} - -static inline void cm_rep_set_flow_ctrl(struct cm_rep_msg *rep_msg, - u8 flow_ctrl) -{ - rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0xFE) | - (flow_ctrl & 0x1)); -} - -static inline u8 cm_rep_get_rnr_retry_count(struct cm_rep_msg *rep_msg) -{ - return (u8) (rep_msg->offset27 >> 5); -} - -static inline void cm_rep_set_rnr_retry_count(struct cm_rep_msg *rep_msg, - u8 rnr_retry_count) -{ - rep_msg->offset27 = (u8) ((rep_msg->offset27 & 0x1F) | - (rnr_retry_count << 5)); -} - -static inline u8 cm_rep_get_srq(struct cm_rep_msg *rep_msg) -{ - return (u8) ((rep_msg->offset27 >> 4) & 0x1); -} - -static inline void cm_rep_set_srq(struct cm_rep_msg *rep_msg, u8 srq) -{ - rep_msg->offset27 = (u8) ((rep_msg->offset27 & 0xEF) | - ((srq & 0x1) << 4)); -} - -struct cm_rtu_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - u8 private_data[IB_CM_RTU_PRIVATE_DATA_SIZE]; - -} __packed; - -struct cm_dreq_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - /* remote QPN/EECN:24, rsvd:8 */ - __be32 offset8; - - u8 private_data[IB_CM_DREQ_PRIVATE_DATA_SIZE]; - -} __packed; - -static inline __be32 cm_dreq_get_remote_qpn(struct cm_dreq_msg *dreq_msg) -{ - return cpu_to_be32(be32_to_cpu(dreq_msg->offset8) >> 8); -} - -static inline void cm_dreq_set_remote_qpn(struct cm_dreq_msg *dreq_msg, __be32 qpn) -{ - dreq_msg->offset8 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(dreq_msg->offset8) & 0x000000FF)); -} - -struct cm_drep_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - u8 private_data[IB_CM_DREP_PRIVATE_DATA_SIZE]; - -} __packed; - -struct cm_lap_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - __be32 rsvd8; - /* remote QPN/EECN:24, remote CM response timeout:5, rsvd:3 */ - __be32 offset12; - __be32 rsvd16; - - __be16 alt_local_lid; - __be16 alt_remote_lid; - union ib_gid alt_local_gid; - union ib_gid alt_remote_gid; - /* flow label:20, rsvd:4, traffic class:8 */ - __be32 offset56; - u8 alt_hop_limit; - /* rsvd:2, packet rate:6 */ - u8 offset61; - /* SL:4, subnet local:1, rsvd:3 */ - u8 offset62; - /* local ACK timeout:5, rsvd:3 */ - u8 offset63; - - u8 private_data[IB_CM_LAP_PRIVATE_DATA_SIZE]; -} __packed; - -static inline __be32 cm_lap_get_remote_qpn(struct cm_lap_msg *lap_msg) -{ - return cpu_to_be32(be32_to_cpu(lap_msg->offset12) >> 8); -} - -static inline void cm_lap_set_remote_qpn(struct cm_lap_msg *lap_msg, __be32 qpn) -{ - lap_msg->offset12 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(lap_msg->offset12) & - 0x000000FF)); -} - -static inline u8 cm_lap_get_remote_resp_timeout(struct cm_lap_msg *lap_msg) -{ - return (u8) ((be32_to_cpu(lap_msg->offset12) & 0xF8) >> 3); -} - -static inline void cm_lap_set_remote_resp_timeout(struct cm_lap_msg *lap_msg, - u8 resp_timeout) -{ - lap_msg->offset12 = cpu_to_be32((resp_timeout << 3) | - (be32_to_cpu(lap_msg->offset12) & - 0xFFFFFF07)); -} - -static inline __be32 cm_lap_get_flow_label(struct cm_lap_msg *lap_msg) -{ - return cpu_to_be32(be32_to_cpu(lap_msg->offset56) >> 12); -} - -static inline void cm_lap_set_flow_label(struct cm_lap_msg *lap_msg, - __be32 flow_label) -{ - lap_msg->offset56 = cpu_to_be32( - (be32_to_cpu(lap_msg->offset56) & 0x00000FFF) | - (be32_to_cpu(flow_label) << 12)); -} - -static inline u8 cm_lap_get_traffic_class(struct cm_lap_msg *lap_msg) -{ - return (u8) be32_to_cpu(lap_msg->offset56); -} - -static inline void cm_lap_set_traffic_class(struct cm_lap_msg *lap_msg, - u8 traffic_class) -{ - lap_msg->offset56 = cpu_to_be32(traffic_class | - (be32_to_cpu(lap_msg->offset56) & - 0xFFFFFF00)); -} - -static inline u8 cm_lap_get_packet_rate(struct cm_lap_msg *lap_msg) -{ - return lap_msg->offset61 & 0x3F; -} - -static inline void cm_lap_set_packet_rate(struct cm_lap_msg *lap_msg, - u8 packet_rate) -{ - lap_msg->offset61 = (packet_rate & 0x3F) | (lap_msg->offset61 & 0xC0); -} - -static inline u8 cm_lap_get_sl(struct cm_lap_msg *lap_msg) -{ - return lap_msg->offset62 >> 4; -} - -static inline void cm_lap_set_sl(struct cm_lap_msg *lap_msg, u8 sl) -{ - lap_msg->offset62 = (sl << 4) | (lap_msg->offset62 & 0x0F); -} - -static inline u8 cm_lap_get_subnet_local(struct cm_lap_msg *lap_msg) -{ - return (lap_msg->offset62 >> 3) & 0x1; -} - -static inline void cm_lap_set_subnet_local(struct cm_lap_msg *lap_msg, - u8 subnet_local) -{ - lap_msg->offset62 = ((subnet_local & 0x1) << 3) | - (lap_msg->offset61 & 0xF7); -} -static inline u8 cm_lap_get_local_ack_timeout(struct cm_lap_msg *lap_msg) -{ - return lap_msg->offset63 >> 3; -} - -static inline void cm_lap_set_local_ack_timeout(struct cm_lap_msg *lap_msg, - u8 local_ack_timeout) -{ - lap_msg->offset63 = (local_ack_timeout << 3) | - (lap_msg->offset63 & 0x07); -} - -struct cm_apr_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - u8 info_length; - u8 ap_status; - __be16 rsvd; - u8 info[IB_CM_APR_INFO_LENGTH]; - - u8 private_data[IB_CM_APR_PRIVATE_DATA_SIZE]; -} __packed; - -struct cm_sidr_req_msg { - struct ib_mad_hdr hdr; - - __be32 request_id; - __be16 pkey; - __be16 rsvd; - __be64 service_id; - - u32 private_data[IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE / sizeof(u32)]; -} __packed; - -struct cm_sidr_rep_msg { - struct ib_mad_hdr hdr; - - __be32 request_id; - u8 status; - u8 info_length; - __be16 rsvd; - /* QPN:24, rsvd:8 */ - __be32 offset8; - __be64 service_id; - __be32 qkey; - u8 info[IB_CM_SIDR_REP_INFO_LENGTH]; - - u8 private_data[IB_CM_SIDR_REP_PRIVATE_DATA_SIZE]; -} __packed; - -static inline __be32 cm_sidr_rep_get_qpn(struct cm_sidr_rep_msg *sidr_rep_msg) -{ - return cpu_to_be32(be32_to_cpu(sidr_rep_msg->offset8) >> 8); -} - -static inline void cm_sidr_rep_set_qpn(struct cm_sidr_rep_msg *sidr_rep_msg, - __be32 qpn) -{ - sidr_rep_msg->offset8 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(sidr_rep_msg->offset8) & - 0x000000FF)); + cpu_to_be32(IBA_GET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, + rep_msg)) : + cpu_to_be32(IBA_GET(CM_REP_LOCAL_QPN, rep_msg)); } #endif /* CM_MSGS_H */ diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 43a6f07e0afe..72f032160c4b 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -36,6 +36,7 @@ #include "core_priv.h" #include "cma_priv.h" +#include "cma_trace.h" MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("Generic RDMA CM Agent"); @@ -877,6 +878,7 @@ struct rdma_cm_id *__rdma_create_id(struct net *net, id_priv->id.route.addr.dev_addr.net = get_net(net); id_priv->seq_num &= 0x00ffffff; + trace_cm_id_create(id_priv); return &id_priv->id; } EXPORT_SYMBOL(__rdma_create_id); @@ -928,27 +930,34 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, int ret; id_priv = container_of(id, struct rdma_id_private, id); - if (id->device != pd->device) - return -EINVAL; + if (id->device != pd->device) { + ret = -EINVAL; + goto out_err; + } qp_init_attr->port_num = id->port_num; qp = ib_create_qp(pd, qp_init_attr); - if (IS_ERR(qp)) - return PTR_ERR(qp); + if (IS_ERR(qp)) { + ret = PTR_ERR(qp); + goto out_err; + } if (id->qp_type == IB_QPT_UD) ret = cma_init_ud_qp(id_priv, qp); else ret = cma_init_conn_qp(id_priv, qp); if (ret) - goto err; + goto out_destroy; id->qp = qp; id_priv->qp_num = qp->qp_num; id_priv->srq = (qp->srq != NULL); + trace_cm_qp_create(id_priv, pd, qp_init_attr, 0); return 0; -err: +out_destroy: ib_destroy_qp(qp); +out_err: + trace_cm_qp_create(id_priv, pd, qp_init_attr, ret); return ret; } EXPORT_SYMBOL(rdma_create_qp); @@ -958,6 +967,7 @@ void rdma_destroy_qp(struct rdma_cm_id *id) struct rdma_id_private *id_priv; id_priv = container_of(id, struct rdma_id_private, id); + trace_cm_qp_destroy(id_priv); mutex_lock(&id_priv->qp_mutex); ib_destroy_qp(id_priv->id.qp); id_priv->id.qp = NULL; @@ -1811,6 +1821,7 @@ void rdma_destroy_id(struct rdma_cm_id *id) enum rdma_cm_state state; id_priv = container_of(id, struct rdma_id_private, id); + trace_cm_id_destroy(id_priv); state = cma_exch(id_priv, RDMA_CM_DESTROYING); cma_cancel_operation(id_priv, state); @@ -1863,6 +1874,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv) if (ret) goto reject; + trace_cm_send_rtu(id_priv); ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); if (ret) goto reject; @@ -1871,6 +1883,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv) reject: pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret); cma_modify_qp_err(id_priv); + trace_cm_send_rej(id_priv); ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, NULL, 0); return ret; @@ -1890,6 +1903,17 @@ static void cma_set_rep_event_data(struct rdma_cm_event *event, event->param.conn.qp_num = rep_data->remote_qpn; } +static int cma_cm_event_handler(struct rdma_id_private *id_priv, + struct rdma_cm_event *event) +{ + int ret; + + trace_cm_event_handler(id_priv, event); + ret = id_priv->id.event_handler(&id_priv->id, event); + trace_cm_event_done(id_priv, event, ret); + return ret; +} + static int cma_ib_handler(struct ib_cm_id *cm_id, const struct ib_cm_event *ib_event) { @@ -1912,8 +1936,10 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, break; case IB_CM_REP_RECEIVED: if (cma_comp(id_priv, RDMA_CM_CONNECT) && - (id_priv->id.qp_type != IB_QPT_UD)) + (id_priv->id.qp_type != IB_QPT_UD)) { + trace_cm_send_mra(id_priv); ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); + } if (id_priv->id.qp) { event.status = cma_rep_recv(id_priv); event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : @@ -1958,7 +1984,7 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, goto out; } - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.ib = NULL; @@ -2119,6 +2145,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, if (IS_ERR(listen_id)) return PTR_ERR(listen_id); + trace_cm_req_handler(listen_id, ib_event->event); if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) { ret = -EINVAL; goto net_dev_put; @@ -2161,7 +2188,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, * until we're done accessing it. */ atomic_inc(&conn_id->refcount); - ret = conn_id->id.event_handler(&conn_id->id, &event); + ret = cma_cm_event_handler(conn_id, &event); if (ret) goto err3; /* @@ -2170,8 +2197,10 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, */ mutex_lock(&lock); if (cma_comp(conn_id, RDMA_CM_CONNECT) && - (conn_id->id.qp_type != IB_QPT_UD)) + (conn_id->id.qp_type != IB_QPT_UD)) { + trace_cm_send_mra(cm_id->context); ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); + } mutex_unlock(&lock); mutex_unlock(&conn_id->handler_mutex); mutex_unlock(&listen_id->handler_mutex); @@ -2286,7 +2315,7 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) event.status = iw_event->status; event.param.conn.private_data = iw_event->private_data; event.param.conn.private_data_len = iw_event->private_data_len; - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.iw = NULL; @@ -2363,7 +2392,7 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, * until we're done accessing it. */ atomic_inc(&conn_id->refcount); - ret = conn_id->id.event_handler(&conn_id->id, &event); + ret = cma_cm_event_handler(conn_id, &event); if (ret) { /* User wants to destroy the CM ID */ conn_id->cm_id.iw = NULL; @@ -2435,6 +2464,7 @@ static int cma_listen_handler(struct rdma_cm_id *id, id->context = id_priv->id.context; id->event_handler = id_priv->id.event_handler; + trace_cm_event_handler(id_priv, event); return id_priv->id.event_handler(id, event); } @@ -2611,7 +2641,7 @@ static void cma_work_handler(struct work_struct *_work) if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) goto out; - if (id_priv->id.event_handler(&id_priv->id, &work->event)) { + if (cma_cm_event_handler(id_priv, &work->event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); destroy = 1; } @@ -2634,7 +2664,7 @@ static void cma_ndev_work_handler(struct work_struct *_work) id_priv->state == RDMA_CM_DEVICE_REMOVAL) goto out; - if (id_priv->id.event_handler(&id_priv->id, &work->event)) { + if (cma_cm_event_handler(id_priv, &work->event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); destroy = 1; } @@ -3089,7 +3119,7 @@ static void addr_handler(int status, struct sockaddr *src_addr, } else event.event = RDMA_CM_EVENT_ADDR_RESOLVED; - if (id_priv->id.event_handler(&id_priv->id, &event)) { + if (cma_cm_event_handler(id_priv, &event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); @@ -3118,6 +3148,7 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv) rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); + atomic_inc(&id_priv->refcount); cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; @@ -3144,6 +3175,7 @@ static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); + atomic_inc(&id_priv->refcount); cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; @@ -3736,7 +3768,7 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, goto out; } - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); rdma_destroy_ah_attr(&event.param.ud.ah_attr); if (ret) { @@ -3800,6 +3832,7 @@ static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); req.max_cm_retries = CMA_MAX_CM_RETRIES; + trace_cm_send_sidr_req(id_priv); ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); if (ret) { ib_destroy_cm_id(id_priv->cm_id.ib); @@ -3873,6 +3906,7 @@ static int cma_connect_ib(struct rdma_id_private *id_priv, req.max_cm_retries = CMA_MAX_CM_RETRIES; req.srq = id_priv->srq ? 1 : 0; + trace_cm_send_req(id_priv); ret = ib_send_cm_req(id_priv->cm_id.ib, &req); out: if (ret && !IS_ERR(id)) { @@ -3986,6 +4020,7 @@ static int cma_accept_ib(struct rdma_id_private *id_priv, rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); rep.srq = id_priv->srq ? 1 : 0; + trace_cm_send_rep(id_priv); ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); out: return ret; @@ -4035,6 +4070,7 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv, rep.private_data = private_data; rep.private_data_len = private_data_len; + trace_cm_send_sidr_rep(id_priv); return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); } @@ -4120,13 +4156,15 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data, return -EINVAL; if (rdma_cap_ib_cm(id->device, id->port_num)) { - if (id->qp_type == IB_QPT_UD) + if (id->qp_type == IB_QPT_UD) { ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0, private_data, private_data_len); - else + } else { + trace_cm_send_rej(id_priv); ret = ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, private_data, private_data_len); + } } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_reject(id_priv->cm_id.iw, private_data, private_data_len); @@ -4151,8 +4189,13 @@ int rdma_disconnect(struct rdma_cm_id *id) if (ret) goto out; /* Initiate or respond to a disconnect. */ - if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) - ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); + trace_cm_disconnect(id_priv); + if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { + if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) + trace_cm_sent_drep(id_priv); + } else { + trace_cm_sent_dreq(id_priv); + } } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); } else @@ -4218,7 +4261,7 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) } else event.event = RDMA_CM_EVENT_MULTICAST_ERROR; - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); rdma_destroy_ah_attr(&event.param.ud.ah_attr); if (ret) { @@ -4623,6 +4666,7 @@ static void cma_add_one(struct ib_device *device) cma_listen_on_dev(id_priv, cma_dev); mutex_unlock(&lock); + trace_cm_add_one(device); return; free_gid_type: @@ -4653,7 +4697,7 @@ static int cma_remove_id_dev(struct rdma_id_private *id_priv) goto out; event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); out: mutex_unlock(&id_priv->handler_mutex); return ret; @@ -4691,6 +4735,8 @@ static void cma_remove_one(struct ib_device *device, void *client_data) { struct cma_device *cma_dev = client_data; + trace_cm_remove_one(device); + if (!cma_dev) return; diff --git a/drivers/infiniband/core/cma_trace.c b/drivers/infiniband/core/cma_trace.c new file mode 100644 index 000000000000..b314a281e10e --- /dev/null +++ b/drivers/infiniband/core/cma_trace.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Trace points for the RDMA Connection Manager. + * + * Author: Chuck Lever <chuck.lever@oracle.com> + * + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + */ + +#define CREATE_TRACE_POINTS + +#include <rdma/rdma_cm.h> +#include <rdma/ib_cm.h> +#include "cma_priv.h" + +#include "cma_trace.h" diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h new file mode 100644 index 000000000000..81e36bf13159 --- /dev/null +++ b/drivers/infiniband/core/cma_trace.h @@ -0,0 +1,391 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Trace point definitions for the RDMA Connect Manager. + * + * Author: Chuck Lever <chuck.lever@oracle.com> + * + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rdma_cma + +#if !defined(_TRACE_RDMA_CMA_H) || defined(TRACE_HEADER_MULTI_READ) + +#define _TRACE_RDMA_CMA_H + +#include <linux/tracepoint.h> +#include <trace/events/rdma.h> + +/* + * enum ib_cm_event_type, from include/rdma/ib_cm.h + */ +#define IB_CM_EVENT_LIST \ + ib_cm_event(REQ_ERROR) \ + ib_cm_event(REQ_RECEIVED) \ + ib_cm_event(REP_ERROR) \ + ib_cm_event(REP_RECEIVED) \ + ib_cm_event(RTU_RECEIVED) \ + ib_cm_event(USER_ESTABLISHED) \ + ib_cm_event(DREQ_ERROR) \ + ib_cm_event(DREQ_RECEIVED) \ + ib_cm_event(DREP_RECEIVED) \ + ib_cm_event(TIMEWAIT_EXIT) \ + ib_cm_event(MRA_RECEIVED) \ + ib_cm_event(REJ_RECEIVED) \ + ib_cm_event(LAP_ERROR) \ + ib_cm_event(LAP_RECEIVED) \ + ib_cm_event(APR_RECEIVED) \ + ib_cm_event(SIDR_REQ_ERROR) \ + ib_cm_event(SIDR_REQ_RECEIVED) \ + ib_cm_event_end(SIDR_REP_RECEIVED) + +#undef ib_cm_event +#undef ib_cm_event_end + +#define ib_cm_event(x) TRACE_DEFINE_ENUM(IB_CM_##x); +#define ib_cm_event_end(x) TRACE_DEFINE_ENUM(IB_CM_##x); + +IB_CM_EVENT_LIST + +#undef ib_cm_event +#undef ib_cm_event_end + +#define ib_cm_event(x) { IB_CM_##x, #x }, +#define ib_cm_event_end(x) { IB_CM_##x, #x } + +#define rdma_show_ib_cm_event(x) \ + __print_symbolic(x, IB_CM_EVENT_LIST) + + +DECLARE_EVENT_CLASS(cma_fsm_class, + TP_PROTO( + const struct rdma_id_private *id_priv + ), + + TP_ARGS(id_priv), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos + ) +); + +#define DEFINE_CMA_FSM_EVENT(name) \ + DEFINE_EVENT(cma_fsm_class, cm_##name, \ + TP_PROTO( \ + const struct rdma_id_private *id_priv \ + ), \ + TP_ARGS(id_priv)) + +DEFINE_CMA_FSM_EVENT(send_rtu); +DEFINE_CMA_FSM_EVENT(send_rej); +DEFINE_CMA_FSM_EVENT(send_mra); +DEFINE_CMA_FSM_EVENT(send_sidr_req); +DEFINE_CMA_FSM_EVENT(send_sidr_rep); +DEFINE_CMA_FSM_EVENT(disconnect); +DEFINE_CMA_FSM_EVENT(sent_drep); +DEFINE_CMA_FSM_EVENT(sent_dreq); +DEFINE_CMA_FSM_EVENT(id_destroy); + +TRACE_EVENT(cm_id_create, + TP_PROTO( + const struct rdma_id_private *id_priv + ), + + TP_ARGS(id_priv), + + TP_STRUCT__entry( + __field(u32, cm_id) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + ), + + TP_printk("cm.id=%u", + __entry->cm_id + ) +); + +DECLARE_EVENT_CLASS(cma_qp_class, + TP_PROTO( + const struct rdma_id_private *id_priv + ), + + TP_ARGS(id_priv), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(u32, qp_num) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->qp_num = id_priv->qp_num; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u qp_num=%u", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + __entry->qp_num + ) +); + +#define DEFINE_CMA_QP_EVENT(name) \ + DEFINE_EVENT(cma_qp_class, cm_##name, \ + TP_PROTO( \ + const struct rdma_id_private *id_priv \ + ), \ + TP_ARGS(id_priv)) + +DEFINE_CMA_QP_EVENT(send_req); +DEFINE_CMA_QP_EVENT(send_rep); +DEFINE_CMA_QP_EVENT(qp_destroy); + +/* + * enum ib_wp_type, from include/rdma/ib_verbs.h + */ +#define IB_QP_TYPE_LIST \ + ib_qp_type(SMI) \ + ib_qp_type(GSI) \ + ib_qp_type(RC) \ + ib_qp_type(UC) \ + ib_qp_type(UD) \ + ib_qp_type(RAW_IPV6) \ + ib_qp_type(RAW_ETHERTYPE) \ + ib_qp_type(RAW_PACKET) \ + ib_qp_type(XRC_INI) \ + ib_qp_type_end(XRC_TGT) + +#undef ib_qp_type +#undef ib_qp_type_end + +#define ib_qp_type(x) TRACE_DEFINE_ENUM(IB_QPT_##x); +#define ib_qp_type_end(x) TRACE_DEFINE_ENUM(IB_QPT_##x); + +IB_QP_TYPE_LIST + +#undef ib_qp_type +#undef ib_qp_type_end + +#define ib_qp_type(x) { IB_QPT_##x, #x }, +#define ib_qp_type_end(x) { IB_QPT_##x, #x } + +#define rdma_show_qp_type(x) \ + __print_symbolic(x, IB_QP_TYPE_LIST) + + +TRACE_EVENT(cm_qp_create, + TP_PROTO( + const struct rdma_id_private *id_priv, + const struct ib_pd *pd, + const struct ib_qp_init_attr *qp_init_attr, + int rc + ), + + TP_ARGS(id_priv, pd, qp_init_attr, rc), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, pd_id) + __field(u32, tos) + __field(u32, qp_num) + __field(u32, send_wr) + __field(u32, recv_wr) + __field(int, rc) + __field(unsigned long, qp_type) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->pd_id = pd->res.id; + __entry->tos = id_priv->tos; + __entry->send_wr = qp_init_attr->cap.max_send_wr; + __entry->recv_wr = qp_init_attr->cap.max_recv_wr; + __entry->rc = rc; + if (!rc) { + __entry->qp_num = id_priv->qp_num; + __entry->qp_type = id_priv->id.qp_type; + } else { + __entry->qp_num = 0; + __entry->qp_type = 0; + } + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u pd.id=%u qp_type=%s" + " send_wr=%u recv_wr=%u qp_num=%u rc=%d", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, + __entry->tos, __entry->pd_id, + rdma_show_qp_type(__entry->qp_type), __entry->send_wr, + __entry->recv_wr, __entry->qp_num, __entry->rc + ) +); + +TRACE_EVENT(cm_req_handler, + TP_PROTO( + const struct rdma_id_private *id_priv, + int event + ), + + TP_ARGS(id_priv, event), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(unsigned long, event) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->event = event; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu)", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + rdma_show_ib_cm_event(__entry->event), __entry->event + ) +); + +TRACE_EVENT(cm_event_handler, + TP_PROTO( + const struct rdma_id_private *id_priv, + const struct rdma_cm_event *event + ), + + TP_ARGS(id_priv, event), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(unsigned long, event) + __field(int, status) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->event = event->event; + __entry->status = event->status; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu/%d)", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + rdma_show_cm_event(__entry->event), __entry->event, + __entry->status + ) +); + +TRACE_EVENT(cm_event_done, + TP_PROTO( + const struct rdma_id_private *id_priv, + const struct rdma_cm_event *event, + int result + ), + + TP_ARGS(id_priv, event, result), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(unsigned long, event) + __field(int, result) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->event = event->event; + __entry->result = result; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s consumer returns %d", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + rdma_show_cm_event(__entry->event), __entry->result + ) +); + +DECLARE_EVENT_CLASS(cma_client_class, + TP_PROTO( + const struct ib_device *device + ), + + TP_ARGS(device), + + TP_STRUCT__entry( + __string(name, device->name) + ), + + TP_fast_assign( + __assign_str(name, device->name); + ), + + TP_printk("device name=%s", + __get_str(name) + ) +); + +#define DEFINE_CMA_CLIENT_EVENT(name) \ + DEFINE_EVENT(cma_client_class, cm_##name, \ + TP_PROTO( \ + const struct ib_device *device \ + ), \ + TP_ARGS(device)) + +DEFINE_CMA_CLIENT_EVENT(add_one); +DEFINE_CMA_CLIENT_EVENT(remove_one); + +#endif /* _TRACE_RDMA_CMA_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE cma_trace + +#include <trace/define_trace.h> diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index 3645e092e1c7..b1457b3464d3 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -149,6 +149,7 @@ unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port); int ib_cache_setup_one(struct ib_device *device); void ib_cache_cleanup_one(struct ib_device *device); void ib_cache_release_one(struct ib_device *device); +void ib_dispatch_event_clients(struct ib_event *event); #ifdef CONFIG_CGROUP_RDMA void ib_device_register_rdmacg(struct ib_device *device); @@ -320,7 +321,7 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, struct ib_udata *udata, - struct ib_uobject *uobj) + struct ib_uqp_object *uobj) { enum ib_qp_type qp_type = attr->qp_type; struct ib_qp *qp; diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index bbfded6d5d3d..4f25b2400694 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -7,6 +7,8 @@ #include <linux/slab.h> #include <rdma/ib_verbs.h> +#include <trace/events/rdma_core.h> + /* # of WCs to poll for with a single call to ib_poll_cq */ #define IB_POLL_BATCH 16 #define IB_POLL_BATCH_DIRECT 8 @@ -41,6 +43,7 @@ static void ib_cq_rdma_dim_work(struct work_struct *w) dim->state = DIM_START_MEASURE; + trace_cq_modify(cq, comps, usec); cq->device->ops.modify_cq(cq, comps, usec); } @@ -65,18 +68,29 @@ static void rdma_dim_init(struct ib_cq *cq) INIT_WORK(&dim->work, ib_cq_rdma_dim_work); } +static int __poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc) +{ + int rc; + + rc = ib_poll_cq(cq, num_entries, wc); + trace_cq_poll(cq, num_entries, rc); + return rc; +} + static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs, int batch) { int i, n, completed = 0; + trace_cq_process(cq); + /* * budget might be (-1) if the caller does not * want to bound this call, thus we need unsigned * minimum here. */ - while ((n = ib_poll_cq(cq, min_t(u32, batch, - budget - completed), wcs)) > 0) { + while ((n = __poll_cq(cq, min_t(u32, batch, + budget - completed), wcs)) > 0) { for (i = 0; i < n; i++) { struct ib_wc *wc = &wcs[i]; @@ -131,8 +145,10 @@ static int ib_poll_handler(struct irq_poll *iop, int budget) completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH); if (completed < budget) { irq_poll_complete(&cq->iop); - if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) + if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) { + trace_cq_reschedule(cq); irq_poll_sched(&cq->iop); + } } if (dim) @@ -143,6 +159,7 @@ static int ib_poll_handler(struct irq_poll *iop, int budget) static void ib_cq_completion_softirq(struct ib_cq *cq, void *private) { + trace_cq_schedule(cq); irq_poll_sched(&cq->iop); } @@ -162,6 +179,7 @@ static void ib_cq_poll_work(struct work_struct *work) static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) { + trace_cq_schedule(cq); queue_work(cq->comp_wq, &cq->work); } @@ -239,6 +257,7 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private, goto out_destroy_cq; } + trace_cq_alloc(cq, nr_cqe, comp_vector, poll_ctx); return cq; out_destroy_cq: @@ -248,6 +267,7 @@ out_free_wc: kfree(cq->wc); out_free_cq: kfree(cq); + trace_cq_alloc_error(nr_cqe, comp_vector, poll_ctx, ret); return ERR_PTR(ret); } EXPORT_SYMBOL(__ib_alloc_cq_user); @@ -304,6 +324,7 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) WARN_ON_ONCE(1); } + trace_cq_free(cq); rdma_restrack_del(&cq->res); cq->device->ops.destroy_cq(cq, udata); if (cq->dim) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 84dd74fe13b8..f6c255202d7f 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -587,7 +587,8 @@ struct ib_device *_ib_alloc_device(size_t size) rdma_init_coredev(&device->coredev, device, &init_net); INIT_LIST_HEAD(&device->event_handler_list); - spin_lock_init(&device->event_handler_lock); + spin_lock_init(&device->qp_open_list_lock); + init_rwsem(&device->event_handler_rwsem); mutex_init(&device->unregistration_lock); /* * client_data needs to be alloc because we don't want our mark to be @@ -1931,17 +1932,15 @@ EXPORT_SYMBOL(ib_set_client_data); * * ib_register_event_handler() registers an event handler that will be * called back when asynchronous IB events occur (as defined in - * chapter 11 of the InfiniBand Architecture Specification). This - * callback may occur in interrupt context. + * chapter 11 of the InfiniBand Architecture Specification). This + * callback occurs in workqueue context. */ void ib_register_event_handler(struct ib_event_handler *event_handler) { - unsigned long flags; - - spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); + down_write(&event_handler->device->event_handler_rwsem); list_add_tail(&event_handler->list, &event_handler->device->event_handler_list); - spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); + up_write(&event_handler->device->event_handler_rwsem); } EXPORT_SYMBOL(ib_register_event_handler); @@ -1954,35 +1953,23 @@ EXPORT_SYMBOL(ib_register_event_handler); */ void ib_unregister_event_handler(struct ib_event_handler *event_handler) { - unsigned long flags; - - spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); + down_write(&event_handler->device->event_handler_rwsem); list_del(&event_handler->list); - spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); + up_write(&event_handler->device->event_handler_rwsem); } EXPORT_SYMBOL(ib_unregister_event_handler); -/** - * ib_dispatch_event - Dispatch an asynchronous event - * @event:Event to dispatch - * - * Low-level drivers must call ib_dispatch_event() to dispatch the - * event to all registered event handlers when an asynchronous event - * occurs. - */ -void ib_dispatch_event(struct ib_event *event) +void ib_dispatch_event_clients(struct ib_event *event) { - unsigned long flags; struct ib_event_handler *handler; - spin_lock_irqsave(&event->device->event_handler_lock, flags); + down_read(&event->device->event_handler_rwsem); list_for_each_entry(handler, &event->device->event_handler_list, list) handler->handler(handler, event); - spin_unlock_irqrestore(&event->device->event_handler_lock, flags); + up_read(&event->device->event_handler_rwsem); } -EXPORT_SYMBOL(ib_dispatch_event); static int iw_query_port(struct ib_device *device, u8 port_num, @@ -1990,7 +1977,6 @@ static int iw_query_port(struct ib_device *device, { struct in_device *inetdev; struct net_device *netdev; - int err; memset(port_attr, 0, sizeof(*port_attr)); @@ -2021,11 +2007,7 @@ static int iw_query_port(struct ib_device *device, } dev_put(netdev); - err = device->ops.query_port(device, port_num, port_attr); - if (err) - return err; - - return 0; + return device->ops.query_port(device, port_num, port_attr); } static int __ib_query_port(struct ib_device *device, diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index b7cb59844ece..b51bd7087a88 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -232,7 +232,9 @@ void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry) if (!entry) return; + xa_lock(&entry->ucontext->mmap_xa); entry->driver_removed = true; + xa_unlock(&entry->ucontext->mmap_xa); kref_put(&entry->ref, rdma_user_mmap_entry_free); } EXPORT_SYMBOL(rdma_user_mmap_entry_remove); diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index cbf6041a5d4a..37b433aa7306 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -41,6 +41,7 @@ #include "core_priv.h" #include "cma_priv.h" #include "restrack.h" +#include "uverbs.h" typedef int (*res_fill_func_t)(struct sk_buff*, bool, struct rdma_restrack_entry*, uint32_t); @@ -599,7 +600,7 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin, goto err; if (!rdma_is_kernel_res(res) && nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, - cq->uobject->context->res.id)) + cq->uobject->uevent.uobject.context->res.id)) goto err; if (fill_res_name_pid(msg, res)) diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 6c72773faf29..5128cb16bb48 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -42,26 +42,21 @@ #include "core_priv.h" #include "rdma_core.h" -void uverbs_uobject_get(struct ib_uobject *uobject) -{ - kref_get(&uobject->ref); -} - static void uverbs_uobject_free(struct kref *ref) { - struct ib_uobject *uobj = - container_of(ref, struct ib_uobject, ref); - - if (uobj->uapi_object->type_class->needs_kfree_rcu) - kfree_rcu(uobj, rcu); - else - kfree(uobj); + kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu); } +/* + * In order to indicate we no longer needs this uobject, uverbs_uobject_put + * is called. When the reference count is decreased, the uobject is freed. + * For example, this is used when attaching a completion channel to a CQ. + */ void uverbs_uobject_put(struct ib_uobject *uobject) { kref_put(&uobject->ref, uverbs_uobject_free); } +EXPORT_SYMBOL(uverbs_uobject_put); static int uverbs_try_lock_object(struct ib_uobject *uobj, enum rdma_lookup_mode mode) @@ -135,7 +130,11 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj, lockdep_assert_held(&ufile->hw_destroy_rwsem); assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE); - if (uobj->object) { + if (reason == RDMA_REMOVE_ABORT) { + WARN_ON(!list_empty(&uobj->list)); + WARN_ON(!uobj->context); + uobj->uapi_object->type_class->alloc_abort(uobj); + } else if (uobj->object) { ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason, attrs); if (ret) { @@ -151,12 +150,6 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj, uobj->object = NULL; } - if (reason == RDMA_REMOVE_ABORT) { - WARN_ON(!list_empty(&uobj->list)); - WARN_ON(!uobj->context); - uobj->uapi_object->type_class->alloc_abort(uobj); - } - uobj->context = NULL; /* @@ -263,15 +256,20 @@ int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id, } /* alloc_uobj must be undone by uverbs_destroy_uobject() */ -static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile, +static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs, const struct uverbs_api_object *obj) { + struct ib_uverbs_file *ufile = attrs->ufile; struct ib_uobject *uobj; - struct ib_ucontext *ucontext; - ucontext = ib_uverbs_get_ucontext_file(ufile); - if (IS_ERR(ucontext)) - return ERR_CAST(ucontext); + if (!attrs->context) { + struct ib_ucontext *ucontext = + ib_uverbs_get_ucontext_file(ufile); + + if (IS_ERR(ucontext)) + return ERR_CAST(ucontext); + attrs->context = ucontext; + } uobj = kzalloc(obj->type_attrs->obj_size, GFP_KERNEL); if (!uobj) @@ -281,7 +279,7 @@ static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile, * The object is added to the list in the commit stage. */ uobj->ufile = ufile; - uobj->context = ucontext; + uobj->context = attrs->context; INIT_LIST_HEAD(&uobj->list); uobj->uapi_object = obj; /* @@ -358,9 +356,9 @@ lookup_get_fd_uobject(const struct uverbs_api_object *obj, uobject = f->private_data; /* - * fget(id) ensures we are not currently running uverbs_close_fd, - * and the caller is expected to ensure that uverbs_close_fd is never - * done while a call top lookup is possible. + * fget(id) ensures we are not currently running + * uverbs_uobject_fd_release(), and the caller is expected to ensure + * that release is never done while a call to lookup is possible. */ if (f->f_op != fd_type->fops) { fput(f); @@ -424,12 +422,12 @@ free: static struct ib_uobject * alloc_begin_idr_uobject(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile) + struct uverbs_attr_bundle *attrs) { int ret; struct ib_uobject *uobj; - uobj = alloc_uobj(ufile, obj); + uobj = alloc_uobj(attrs, obj); if (IS_ERR(uobj)) return uobj; @@ -445,7 +443,7 @@ alloc_begin_idr_uobject(const struct uverbs_api_object *obj, return uobj; remove: - xa_erase(&ufile->idr, uobj->id); + xa_erase(&attrs->ufile->idr, uobj->id); uobj_put: uverbs_uobject_put(uobj); return ERR_PTR(ret); @@ -453,31 +451,48 @@ uobj_put: static struct ib_uobject * alloc_begin_fd_uobject(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile) + struct uverbs_attr_bundle *attrs) { + const struct uverbs_obj_fd_type *fd_type = + container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); int new_fd; struct ib_uobject *uobj; + struct file *filp; + + if (WARN_ON(fd_type->fops->release != &uverbs_uobject_fd_release)) + return ERR_PTR(-EINVAL); new_fd = get_unused_fd_flags(O_CLOEXEC); if (new_fd < 0) return ERR_PTR(new_fd); - uobj = alloc_uobj(ufile, obj); - if (IS_ERR(uobj)) { - put_unused_fd(new_fd); - return uobj; + uobj = alloc_uobj(attrs, obj); + if (IS_ERR(uobj)) + goto err_fd; + + /* Note that uverbs_uobject_fd_release() is called during abort */ + filp = anon_inode_getfile(fd_type->name, fd_type->fops, NULL, + fd_type->flags); + if (IS_ERR(filp)) { + uobj = ERR_CAST(filp); + goto err_uobj; } + uobj->object = filp; uobj->id = new_fd; - uobj->ufile = ufile; + return uobj; +err_uobj: + uverbs_uobject_put(uobj); +err_fd: + put_unused_fd(new_fd); return uobj; } struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile, struct uverbs_attr_bundle *attrs) { + struct ib_uverbs_file *ufile = attrs->ufile; struct ib_uobject *ret; if (IS_ERR(obj)) @@ -491,13 +506,11 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, if (!down_read_trylock(&ufile->hw_destroy_rwsem)) return ERR_PTR(-EIO); - ret = obj->type_class->alloc_begin(obj, ufile); + ret = obj->type_class->alloc_begin(obj, attrs); if (IS_ERR(ret)) { up_read(&ufile->hw_destroy_rwsem); return ret; } - if (attrs) - attrs->context = ret->context; return ret; } @@ -544,6 +557,9 @@ static void remove_handle_idr_uobject(struct ib_uobject *uobj) static void alloc_abort_fd_uobject(struct ib_uobject *uobj) { + struct file *filp = uobj->object; + + fput(filp); put_unused_fd(uobj->id); } @@ -553,7 +569,7 @@ static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj, { const struct uverbs_obj_fd_type *fd_type = container_of( uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type); - int ret = fd_type->context_closed(uobj, why); + int ret = fd_type->destroy_object(uobj, why); if (ib_is_destroy_retryable(ret, why, uobj)) return ret; @@ -565,7 +581,7 @@ static void remove_handle_fd_uobject(struct ib_uobject *uobj) { } -static int alloc_commit_idr_uobject(struct ib_uobject *uobj) +static void alloc_commit_idr_uobject(struct ib_uobject *uobj) { struct ib_uverbs_file *ufile = uobj->ufile; void *old; @@ -579,33 +595,14 @@ static int alloc_commit_idr_uobject(struct ib_uobject *uobj) */ old = xa_store(&ufile->idr, uobj->id, uobj, GFP_KERNEL); WARN_ON(old != NULL); - - return 0; } -static int alloc_commit_fd_uobject(struct ib_uobject *uobj) +static void alloc_commit_fd_uobject(struct ib_uobject *uobj) { - const struct uverbs_obj_fd_type *fd_type = container_of( - uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type); int fd = uobj->id; - struct file *filp; - - /* - * The kref for uobj is moved into filp->private data and put in - * uverbs_close_fd(). Once alloc_commit() succeeds uverbs_close_fd() - * must be guaranteed to be called from the provided fops release - * callback. - */ - filp = anon_inode_getfile(fd_type->name, - fd_type->fops, - uobj, - fd_type->flags); - if (IS_ERR(filp)) - return PTR_ERR(filp); - - uobj->object = filp; + struct file *filp = uobj->object; - /* Matching put will be done in uverbs_close_fd() */ + /* Matching put will be done in uverbs_uobject_fd_release() */ kref_get(&uobj->ufile->ref); /* This shouldn't be used anymore. Use the file object instead */ @@ -613,11 +610,10 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj) /* * NOTE: Once we install the file we loose ownership of our kref on - * uobj. It will be put by uverbs_close_fd() + * uobj. It will be put by uverbs_uobject_fd_release() */ + filp->private_data = uobj; fd_install(fd, filp); - - return 0; } /* @@ -625,19 +621,13 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj) * caller can no longer assume uobj is valid. If this function fails it * destroys the uboject, including the attached HW object. */ -int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj, - struct uverbs_attr_bundle *attrs) +void rdma_alloc_commit_uobject(struct ib_uobject *uobj, + struct uverbs_attr_bundle *attrs) { struct ib_uverbs_file *ufile = attrs->ufile; - int ret; /* alloc_commit consumes the uobj kref */ - ret = uobj->uapi_object->type_class->alloc_commit(uobj); - if (ret) { - uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs); - up_read(&ufile->hw_destroy_rwsem); - return ret; - } + uobj->uapi_object->type_class->alloc_commit(uobj); /* kref is held so long as the uobj is on the uobj list. */ uverbs_uobject_get(uobj); @@ -650,8 +640,6 @@ int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj, /* Matches the down_read in rdma_alloc_begin_uobject */ up_read(&ufile->hw_destroy_rwsem); - - return 0; } /* @@ -663,7 +651,6 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, { struct ib_uverbs_file *ufile = uobj->ufile; - uobj->object = NULL; uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs); /* Matches the down_read in rdma_alloc_begin_uobject */ @@ -681,7 +668,10 @@ static void lookup_put_fd_uobject(struct ib_uobject *uobj, struct file *filp = uobj->object; WARN_ON(mode != UVERBS_LOOKUP_READ); - /* This indirectly calls uverbs_close_fd and free the object */ + /* + * This indirectly calls uverbs_uobject_fd_release() and free the + * object + */ fput(filp); } @@ -744,33 +734,32 @@ const struct uverbs_obj_type_class uverbs_idr_class = { .lookup_put = lookup_put_idr_uobject, .destroy_hw = destroy_hw_idr_uobject, .remove_handle = remove_handle_idr_uobject, - /* - * When we destroy an object, we first just lock it for WRITE and - * actually DESTROY it in the finalize stage. So, the problematic - * scenario is when we just started the finalize stage of the - * destruction (nothing was executed yet). Now, the other thread - * fetched the object for READ access, but it didn't lock it yet. - * The DESTROY thread continues and starts destroying the object. - * When the other thread continue - without the RCU, it would - * access freed memory. However, the rcu_read_lock delays the free - * until the rcu_read_lock of the READ operation quits. Since the - * exclusive lock of the object is still taken by the DESTROY flow, the - * READ operation will get -EBUSY and it'll just bail out. - */ - .needs_kfree_rcu = true, }; EXPORT_SYMBOL(uverbs_idr_class); -void uverbs_close_fd(struct file *f) +/* + * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct + * file_operations release method. + */ +int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) { - struct ib_uobject *uobj = f->private_data; - struct ib_uverbs_file *ufile = uobj->ufile; - struct uverbs_attr_bundle attrs = { - .context = uobj->context, - .ufile = ufile, - }; + struct ib_uverbs_file *ufile; + struct ib_uobject *uobj; + + /* + * This can only happen if the fput came from alloc_abort_fd_uobject() + */ + if (!filp->private_data) + return 0; + uobj = filp->private_data; + ufile = uobj->ufile; if (down_read_trylock(&ufile->hw_destroy_rwsem)) { + struct uverbs_attr_bundle attrs = { + .context = uobj->context, + .ufile = ufile, + }; + /* * lookup_get_fd_uobject holds the kref on the struct file any * time a FD uobj is locked, which prevents this release @@ -782,13 +771,14 @@ void uverbs_close_fd(struct file *f) up_read(&ufile->hw_destroy_rwsem); } - /* Matches the get in alloc_begin_fd_uobject */ + /* Matches the get in alloc_commit_fd_uobject() */ kref_put(&ufile->ref, ib_uverbs_release_file); /* Pairs with filp->private_data in alloc_begin_fd_uobject */ uverbs_uobject_put(uobj); + return 0; } -EXPORT_SYMBOL(uverbs_close_fd); +EXPORT_SYMBOL(uverbs_uobject_fd_release); /* * Drop the ucontext off the ufile and completely disconnect it from the @@ -855,9 +845,7 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, } /* - * Destroy the uncontext and every uobject associated with it. If called with - * reason != RDMA_REMOVE_CLOSE this will not return until the destruction has - * been completed and ufile->ucontext is NULL. + * Destroy the uncontext and every uobject associated with it. * * This is internally locked and can be called in parallel from multiple * contexts. @@ -865,22 +853,6 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, enum rdma_remove_reason reason) { - if (reason == RDMA_REMOVE_CLOSE) { - /* - * During destruction we might trigger something that - * synchronously calls release on any file descriptor. For - * this reason all paths that come from file_operations - * release must use try_lock. They can progress knowing that - * there is an ongoing uverbs_destroy_ufile_hw that will clean - * up the driver resources. - */ - if (!mutex_trylock(&ufile->ucontext_lock)) - return; - - } else { - mutex_lock(&ufile->ucontext_lock); - } - down_write(&ufile->hw_destroy_rwsem); /* @@ -909,7 +881,6 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, done: up_write(&ufile->hw_destroy_rwsem); - mutex_unlock(&ufile->ucontext_lock); } const struct uverbs_obj_type_class uverbs_fd_class = { @@ -920,7 +891,6 @@ const struct uverbs_obj_type_class uverbs_fd_class = { .lookup_put = lookup_put_fd_uobject, .destroy_hw = destroy_hw_fd_uobject, .remove_handle = remove_handle_fd_uobject, - .needs_kfree_rcu = false, }; EXPORT_SYMBOL(uverbs_fd_class); @@ -943,19 +913,17 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, return rdma_lookup_get_uobject(obj, attrs->ufile, id, UVERBS_LOOKUP_WRITE, attrs); case UVERBS_ACCESS_NEW: - return rdma_alloc_begin_uobject(obj, attrs->ufile, attrs); + return rdma_alloc_begin_uobject(obj, attrs); default: WARN_ON(true); return ERR_PTR(-EOPNOTSUPP); } } -int uverbs_finalize_object(struct ib_uobject *uobj, - enum uverbs_obj_access access, bool commit, - struct uverbs_attr_bundle *attrs) +void uverbs_finalize_object(struct ib_uobject *uobj, + enum uverbs_obj_access access, bool commit, + struct uverbs_attr_bundle *attrs) { - int ret = 0; - /* * refcounts should be handled at the object level and not at the * uobject level. Refcounts of the objects themselves are done in @@ -975,14 +943,11 @@ int uverbs_finalize_object(struct ib_uobject *uobj, break; case UVERBS_ACCESS_NEW: if (commit) - ret = rdma_alloc_commit_uobject(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); else rdma_alloc_abort_uobject(uobj, attrs); break; default: WARN_ON(true); - ret = -EOPNOTSUPP; } - - return ret; } diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index e63fbda25e1d..33978e0f1262 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -51,29 +51,6 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs); /* - * uverbs_uobject_get is called in order to increase the reference count on - * an uobject. This is useful when a handler wants to keep the uobject's memory - * alive, regardless if this uobject is still alive in the context's objects - * repository. Objects are put via uverbs_uobject_put. - */ -void uverbs_uobject_get(struct ib_uobject *uobject); - -/* - * In order to indicate we no longer needs this uobject, uverbs_uobject_put - * is called. When the reference count is decreased, the uobject is freed. - * For example, this is used when attaching a completion channel to a CQ. - */ -void uverbs_uobject_put(struct ib_uobject *uobject); - -/* Indicate this fd is no longer used by this consumer, but its memory isn't - * necessarily released yet. When the last reference is put, we release the - * memory. After this call is executed, calling uverbs_uobject_get isn't - * allowed. - * This must be called from the release file_operations of the file! - */ -void uverbs_close_fd(struct file *f); - -/* * Get an ib_uobject that corresponds to the given id from ufile, assuming * the object is from the given type. Lock it to the required access when * applicable. @@ -86,24 +63,9 @@ struct ib_uobject * uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, s64 id, struct uverbs_attr_bundle *attrs); -/* - * Note that certain finalize stages could return a status: - * (a) alloc_commit could return a failure if the object is committed at the - * same time when the context is destroyed. - * (b) remove_commit could fail if the object wasn't destroyed successfully. - * Since multiple objects could be finalized in one transaction, it is very NOT - * recommended to have several finalize actions which have side effects. - * For example, it's NOT recommended to have a certain action which has both - * a commit action and a destroy action or two destroy objects in the same - * action. The rule of thumb is to have one destroy or commit action with - * multiple lookups. - * The first non zero return value of finalize_object is returned from this - * function. For example, this could happen when we couldn't destroy an - * object. - */ -int uverbs_finalize_object(struct ib_uobject *uobj, - enum uverbs_obj_access access, bool commit, - struct uverbs_attr_bundle *attrs); +void uverbs_finalize_object(struct ib_uobject *uobj, + enum uverbs_obj_access access, bool commit, + struct uverbs_attr_bundle *attrs); int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx); @@ -189,6 +151,7 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, unsigned int num_attrs); void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile); +extern const struct uapi_definition uverbs_def_obj_async_fd[]; extern const struct uapi_definition uverbs_def_obj_counters[]; extern const struct uapi_definition uverbs_def_obj_cq[]; extern const struct uapi_definition uverbs_def_obj_device[]; diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 8917125ea16d..30d4c126a2db 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -1068,7 +1068,7 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb, } settimeout_out: - return skb->len; + return 0; } static inline int ib_nl_is_good_resolve_resp(const struct nlmsghdr *nlh) @@ -1139,7 +1139,7 @@ int ib_nl_handle_resolve_resp(struct sk_buff *skb, } resp_out: - return skb->len; + return 0; } static void free_sm_ah(struct kref *kref) diff --git a/drivers/infiniband/core/trace.c b/drivers/infiniband/core/trace.c new file mode 100644 index 000000000000..6c3514beac4d --- /dev/null +++ b/drivers/infiniband/core/trace.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Trace points for core RDMA functions. + * + * Author: Chuck Lever <chuck.lever@oracle.com> + * + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + */ + +#define CREATE_TRACE_POINTS + +#include <rdma/ib_verbs.h> + +#include <trace/events/rdma_core.h> diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index c3769a5f096d..06b6125b5ae1 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -166,10 +166,13 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, * for any address. */ mask |= (sg_dma_address(sg) + pgoff) ^ va; - if (i && i != (umem->nmap - 1)) - /* restrict by length as well for interior SGEs */ - mask |= sg_dma_len(sg); va += sg_dma_len(sg) - pgoff; + /* Except for the last entry, the ending iova alignment sets + * the maximum possible page size as the low bits of the iova + * must be zero when starting the next chunk. + */ + if (i != (umem->nmap - 1)) + mask |= va; pgoff = 0; } best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap); diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index a71ce0ae2031..b8c657b28380 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -227,21 +227,10 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device, umem_odp->notifier.ops = ops; umem_odp->page_shift = PAGE_SHIFT; - if (access & IB_ACCESS_HUGETLB) { - struct vm_area_struct *vma; - struct hstate *h; - - down_read(&mm->mmap_sem); - vma = find_vma(mm, ib_umem_start(umem_odp)); - if (!vma || !is_vm_hugetlb_page(vma)) { - up_read(&mm->mmap_sem); - ret = -EINVAL; - goto err_free; - } - h = hstate_vma(vma); - umem_odp->page_shift = huge_page_shift(h); - up_read(&mm->mmap_sem); - } +#ifdef CONFIG_HUGETLB_PAGE + if (access & IB_ACCESS_HUGETLB) + umem_odp->page_shift = HPAGE_SHIFT; +#endif umem_odp->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); ret = ib_init_umem_odp(umem_odp, ops); @@ -251,7 +240,6 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device, err_put_pid: put_pid(umem_odp->tgid); -err_free: kfree(umem_odp); return ERR_PTR(ret); } @@ -424,7 +412,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, while (bcnt > 0) { const size_t gup_num_pages = min_t(size_t, - (bcnt + BIT(page_shift) - 1) >> page_shift, + ALIGN(bcnt, PAGE_SIZE) / PAGE_SIZE, PAGE_SIZE / sizeof(struct page *)); down_read(&owning_mm->mmap_sem); diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 63f7f7db5902..7df71983212d 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -111,7 +111,6 @@ struct ib_uverbs_device { struct srcu_struct disassociate_srcu; struct mutex lists_mutex; /* protect lists */ struct list_head uverbs_file_list; - struct list_head uverbs_events_file_list; struct uverbs_api *uapi; }; @@ -124,10 +123,9 @@ struct ib_uverbs_event_queue { }; struct ib_uverbs_async_event_file { + struct ib_uobject uobj; struct ib_uverbs_event_queue ev_queue; - struct ib_uverbs_file *uverbs_file; - struct kref ref; - struct list_head list; + struct ib_event_handler event_handler; }; struct ib_uverbs_completion_event_file { @@ -144,8 +142,7 @@ struct ib_uverbs_file { * ucontext_lock held */ struct ib_ucontext *ucontext; - struct ib_event_handler event_handler; - struct ib_uverbs_async_event_file *async_file; + struct ib_uverbs_async_event_file *async_file; struct list_head list; /* @@ -183,6 +180,7 @@ struct ib_uverbs_mcast_entry { struct ib_uevent_object { struct ib_uobject uobject; + /* List member for ib_uverbs_async_event_file list */ struct list_head event_list; u32 events_reported; }; @@ -210,25 +208,24 @@ struct ib_uwq_object { }; struct ib_ucq_object { - struct ib_uobject uobject; + struct ib_uevent_object uevent; struct list_head comp_list; - struct list_head async_list; u32 comp_events_reported; - u32 async_events_reported; }; extern const struct file_operations uverbs_event_fops; +extern const struct file_operations uverbs_async_event_fops; void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue); -struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file, - struct ib_device *ib_dev); -void ib_uverbs_free_async_event_file(struct ib_uverbs_file *uverbs_file); +void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file); +void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue); void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res); -void ib_uverbs_release_ucq(struct ib_uverbs_file *file, - struct ib_uverbs_completion_event_file *ev_file, +int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs); +int ib_init_ucontext(struct uverbs_attr_bundle *attrs); + +void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file, struct ib_ucq_object *uobj); -void ib_uverbs_release_uevent(struct ib_uverbs_file *file, - struct ib_uevent_object *uobj); +void ib_uverbs_release_uevent(struct ib_uevent_object *uobj); void ib_uverbs_release_file(struct kref *ref); void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context); @@ -236,8 +233,6 @@ void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr); -void ib_uverbs_event_handler(struct ib_event_handler *handler, - struct ib_event *event); int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs); @@ -276,23 +271,6 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type, size_t kern_filter_sz, union ib_flow_spec *ib_spec); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DEVICE); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_PD); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MR); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COMP_CHANNEL); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_CQ); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_QP); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_AH); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MW); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_SRQ); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_WQ); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_RWQ_IND_TBL); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_XRCD); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW_ACTION); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DM); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COUNTERS); - /* * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the * PortInfo CapabilityMask, but was extended with unique bits. diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 06ed32c8662f..c8693f5231dd 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -203,82 +203,55 @@ _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs) #define ib_uverbs_lookup_comp_file(_fd, _ufile) \ _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile) -static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) +int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs) { - struct ib_uverbs_file *file = attrs->ufile; - struct ib_uverbs_get_context cmd; - struct ib_uverbs_get_context_resp resp; - struct ib_ucontext *ucontext; - struct file *filp; - struct ib_rdmacg_object cg_obj; + struct ib_uverbs_file *ufile = attrs->ufile; + struct ib_ucontext *ucontext; struct ib_device *ib_dev; - int ret; - ret = uverbs_request(attrs, &cmd, sizeof(cmd)); - if (ret) - return ret; - - mutex_lock(&file->ucontext_lock); - ib_dev = srcu_dereference(file->device->ib_dev, - &file->device->disassociate_srcu); - if (!ib_dev) { - ret = -EIO; - goto err; - } - - if (file->ucontext) { - ret = -EINVAL; - goto err; - } - - ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); - if (ret) - goto err; + ib_dev = srcu_dereference(ufile->device->ib_dev, + &ufile->device->disassociate_srcu); + if (!ib_dev) + return -EIO; ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext); - if (!ucontext) { - ret = -ENOMEM; - goto err_alloc; - } - - attrs->context = ucontext; + if (!ucontext) + return -ENOMEM; ucontext->res.type = RDMA_RESTRACK_CTX; ucontext->device = ib_dev; - ucontext->cg_obj = cg_obj; - /* ufile is required when some objects are released */ - ucontext->ufile = file; - - ucontext->closing = false; - ucontext->cleanup_retryable = false; - + ucontext->ufile = ufile; xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC); + attrs->context = ucontext; + return 0; +} - ret = get_unused_fd_flags(O_CLOEXEC); - if (ret < 0) - goto err_free; - resp.async_fd = ret; +int ib_init_ucontext(struct uverbs_attr_bundle *attrs) +{ + struct ib_ucontext *ucontext = attrs->context; + struct ib_uverbs_file *file = attrs->ufile; + int ret; - filp = ib_uverbs_alloc_async_event_file(file, ib_dev); - if (IS_ERR(filp)) { - ret = PTR_ERR(filp); - goto err_fd; + if (!down_read_trylock(&file->hw_destroy_rwsem)) + return -EIO; + mutex_lock(&file->ucontext_lock); + if (file->ucontext) { + ret = -EINVAL; + goto err; } - resp.num_comp_vectors = file->device->num_comp_vectors; - - ret = uverbs_response(attrs, &resp, sizeof(resp)); + ret = ib_rdmacg_try_charge(&ucontext->cg_obj, ucontext->device, + RDMACG_RESOURCE_HCA_HANDLE); if (ret) - goto err_file; + goto err; - ret = ib_dev->ops.alloc_ucontext(ucontext, &attrs->driver_udata); + ret = ucontext->device->ops.alloc_ucontext(ucontext, + &attrs->driver_udata); if (ret) - goto err_file; + goto err_uncharge; rdma_restrack_uadd(&ucontext->res); - fd_install(resp.async_fd, filp); - /* * Make sure that ib_uverbs_get_ucontext() sees the pointer update * only after all writes to setup the ucontext have completed @@ -286,24 +259,62 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) smp_store_release(&file->ucontext, ucontext); mutex_unlock(&file->ucontext_lock); - + up_read(&file->hw_destroy_rwsem); return 0; -err_file: - ib_uverbs_free_async_event_file(file); - fput(filp); +err_uncharge: + ib_rdmacg_uncharge(&ucontext->cg_obj, ucontext->device, + RDMACG_RESOURCE_HCA_HANDLE); +err: + mutex_unlock(&file->ucontext_lock); + up_read(&file->hw_destroy_rwsem); + return ret; +} -err_fd: - put_unused_fd(resp.async_fd); +static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) +{ + struct ib_uverbs_get_context_resp resp; + struct ib_uverbs_get_context cmd; + struct ib_device *ib_dev; + struct ib_uobject *uobj; + int ret; -err_free: - kfree(ucontext); + ret = uverbs_request(attrs, &cmd, sizeof(cmd)); + if (ret) + return ret; -err_alloc: - ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); + ret = ib_alloc_ucontext(attrs); + if (ret) + return ret; -err: - mutex_unlock(&file->ucontext_lock); + uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev); + if (IS_ERR(uobj)) { + ret = PTR_ERR(uobj); + goto err_ucontext; + } + + resp = (struct ib_uverbs_get_context_resp){ + .num_comp_vectors = attrs->ufile->device->num_comp_vectors, + .async_fd = uobj->id, + }; + ret = uverbs_response(attrs, &resp, sizeof(resp)); + if (ret) + goto err_uobj; + + ret = ib_init_ucontext(attrs); + if (ret) + goto err_uobj; + + ib_uverbs_init_async_event_file( + container_of(uobj, struct ib_uverbs_async_event_file, uobj)); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; + +err_uobj: + rdma_alloc_abort_uobject(uobj, attrs); +err_ucontext: + kfree(attrs->context); + attrs->context = NULL; return ret; } @@ -446,7 +457,8 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) if (ret) goto err_copy; - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs)); @@ -642,7 +654,8 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) mutex_unlock(&ibudev->xrcd_tree_mutex); - return uobj_alloc_commit(&obj->uobject, attrs); + rdma_alloc_commit_uobject(&obj->uobject, attrs); + return 0; err_copy: if (inode) { @@ -774,7 +787,8 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) uobj_put_obj_read(pd); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs)); @@ -928,7 +942,8 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) goto err_copy; uobj_put_obj_read(pd); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: uverbs_dealloc_mw(mw); @@ -980,7 +995,8 @@ static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs) return ret; } - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; } static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, @@ -1010,11 +1026,9 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, } } - obj->uobject.user_handle = cmd->user_handle; - obj->comp_events_reported = 0; - obj->async_events_reported = 0; + obj->uevent.uobject.user_handle = cmd->user_handle; INIT_LIST_HEAD(&obj->comp_list); - INIT_LIST_HEAD(&obj->async_list); + INIT_LIST_HEAD(&obj->uevent.event_list); attr.cqe = cmd->cqe; attr.comp_vector = cmd->comp_vector; @@ -1026,7 +1040,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, goto err_file; } cq->device = ib_dev; - cq->uobject = &obj->uobject; + cq->uobject = obj; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; @@ -1036,9 +1050,9 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, if (ret) goto err_free; - obj->uobject.object = cq; + obj->uevent.uobject.object = cq; memset(&resp, 0, sizeof resp); - resp.base.cq_handle = obj->uobject.id; + resp.base.cq_handle = obj->uevent.uobject.id; resp.base.cqe = cq->cqe; resp.response_length = uverbs_response_length(attrs, sizeof(resp)); @@ -1049,9 +1063,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, if (ret) goto err_cb; - ret = uobj_alloc_commit(&obj->uobject, attrs); - if (ret) - return ERR_PTR(ret); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); return obj; err_cb: @@ -1061,10 +1073,10 @@ err_free: kfree(cq); err_file: if (ev_file) - ib_uverbs_release_ucq(attrs->ufile, ev_file, obj); + ib_uverbs_release_ucq(ev_file, obj); err: - uobj_alloc_abort(&obj->uobject, attrs); + uobj_alloc_abort(&obj->uevent.uobject, attrs); return ERR_PTR(ret); } @@ -1133,7 +1145,8 @@ static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) ret = uverbs_response(attrs, &resp, sizeof(resp)); out: - uobj_put_obj_read(cq); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -1216,7 +1229,8 @@ static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs) ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT); out_put: - uobj_put_obj_read(cq); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -1237,8 +1251,8 @@ static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs) ib_req_notify_cq(cq, cmd.solicited_only ? IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); - uobj_put_obj_read(cq); - + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return 0; } @@ -1258,10 +1272,10 @@ static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs) if (IS_ERR(uobj)) return PTR_ERR(uobj); - obj = container_of(uobj, struct ib_ucq_object, uobject); + obj = container_of(uobj, struct ib_ucq_object, uevent.uobject); memset(&resp, 0, sizeof(resp)); resp.comp_events_reported = obj->comp_events_reported; - resp.async_events_reported = obj->async_events_reported; + resp.async_events_reported = obj->uevent.events_reported; uobj_put_destroy(uobj); @@ -1375,7 +1389,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, } attr.event_handler = ib_uverbs_qp_event_handler; - attr.qp_context = attrs->ufile; attr.send_cq = scq; attr.recv_cq = rcq; attr.srq = srq; @@ -1391,7 +1404,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, attr.cap.max_recv_sge = cmd->max_recv_sge; attr.cap.max_inline_data = cmd->max_inline_data; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); INIT_LIST_HEAD(&obj->mcast_list); @@ -1421,7 +1433,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, qp = ib_create_qp(pd, &attr); else qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, - &obj->uevent.uobject); + obj); if (IS_ERR(qp)) { ret = PTR_ERR(qp); @@ -1439,7 +1451,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, qp->srq = attr.srq; qp->rwq_ind_tbl = ind_tbl; qp->event_handler = attr.event_handler; - qp->qp_context = attr.qp_context; qp->qp_type = attr.qp_type; atomic_set(&qp->usecnt, 0); atomic_inc(&pd->usecnt); @@ -1454,7 +1465,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, atomic_inc(&ind_tbl->usecnt); } else { /* It is done in _ib_create_qp for other QP types */ - qp->uobject = &obj->uevent.uobject; + qp->uobject = obj; } obj->uevent.uobject.object = qp; @@ -1483,15 +1494,19 @@ static int create_qp(struct uverbs_attr_bundle *attrs, if (pd) uobj_put_obj_read(pd); if (scq) - uobj_put_obj_read(scq); + rdma_lookup_put_uobject(&scq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (rcq && rcq != scq) - uobj_put_obj_read(rcq); + rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (srq) - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ind_tbl) uobj_put_obj_read(ind_tbl); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_cb: ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); @@ -1501,11 +1516,14 @@ err_put: if (pd) uobj_put_obj_read(pd); if (scq) - uobj_put_obj_read(scq); + rdma_lookup_put_uobject(&scq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (rcq && rcq != scq) - uobj_put_obj_read(rcq); + rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (srq) - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ind_tbl) uobj_put_obj_read(ind_tbl); @@ -1567,7 +1585,7 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) struct ib_xrcd *xrcd; struct ib_uobject *uninitialized_var(xrcd_uobj); struct ib_qp *qp; - struct ib_qp_open_attr attr; + struct ib_qp_open_attr attr = {}; int ret; struct ib_device *ib_dev; @@ -1593,11 +1611,9 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) } attr.event_handler = ib_uverbs_qp_event_handler; - attr.qp_context = attrs->ufile; attr.qp_num = cmd.qpn; attr.qp_type = cmd.qp_type; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); INIT_LIST_HEAD(&obj->mcast_list); @@ -1620,10 +1636,11 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); atomic_inc(&obj->uxrcd->refcnt); - qp->uobject = &obj->uevent.uobject; + qp->uobject = obj; uobj_put_read(xrcd_uobj); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_destroy: ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); @@ -1684,7 +1701,8 @@ static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs) ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) goto out; @@ -1921,7 +1939,8 @@ static int modify_qp(struct uverbs_attr_bundle *attrs, &attrs->driver_udata); release_qp: - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); out: kfree(attr); @@ -2185,7 +2204,8 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) ret = ret2; out_put: - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); while (wr) { if (is_ud && ud_wr(wr)->ah) @@ -2327,7 +2347,8 @@ static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs) resp.bad_wr = 0; ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) { for (next = wr; next; next = next->next) { ++resp.bad_wr; @@ -2377,7 +2398,8 @@ static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs) resp.bad_wr = 0; ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr); - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) for (next = wr; next; next = next->next) { @@ -2465,7 +2487,8 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs) goto err_copy; uobj_put_obj_read(pd); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: rdma_destroy_ah_user(ah, RDMA_DESTROY_AH_SLEEPABLE, @@ -2507,7 +2530,7 @@ static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs) if (!qp) return -EINVAL; - obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + obj = qp->uobject; mutex_lock(&obj->mcast_lock); list_for_each_entry(mcast, &obj->mcast_list, list) @@ -2534,7 +2557,8 @@ static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs) out_put: mutex_unlock(&obj->mcast_lock); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -2556,7 +2580,7 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) if (!qp) return -EINVAL; - obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + obj = qp->uobject; mutex_lock(&obj->mcast_lock); list_for_each_entry(mcast, &obj->mcast_list, list) @@ -2577,7 +2601,8 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) out_put: mutex_unlock(&obj->mcast_lock); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -2943,7 +2968,6 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) wq_init_attr.wq_type = cmd.wq_type; wq_init_attr.event_handler = ib_uverbs_wq_event_handler; wq_init_attr.create_flags = cmd.create_flags; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata); @@ -2952,7 +2976,7 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) goto err_put_cq; } - wq->uobject = &obj->uevent.uobject; + wq->uobject = obj; obj->uevent.uobject.object = wq; wq->wq_type = wq_init_attr.wq_type; wq->cq = cq; @@ -2962,7 +2986,7 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) atomic_set(&wq->usecnt, 0); atomic_inc(&pd->usecnt); atomic_inc(&cq->usecnt); - wq->uobject = &obj->uevent.uobject; + wq->uobject = obj; obj->uevent.uobject.object = wq; memset(&resp, 0, sizeof(resp)); @@ -2976,13 +3000,16 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) goto err_copy; uobj_put_obj_read(pd); - uobj_put_obj_read(cq); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_copy: ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs)); err_put_cq: - uobj_put_obj_read(cq); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_put_pd: uobj_put_obj_read(pd); err_uobj: @@ -3048,7 +3075,8 @@ static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs) } ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask, &attrs->driver_udata); - uobj_put_obj_read(wq); + rdma_lookup_put_uobject(&wq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -3149,9 +3177,11 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) kfree(wqs_handles); for (j = 0; j < num_read_wqs; j++) - uobj_put_obj_read(wqs[j]); + rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: ib_destroy_rwq_ind_table(rwq_ind_tbl); @@ -3159,7 +3189,8 @@ err_uobj: uobj_alloc_abort(uobj, attrs); put_wqs: for (j = 0; j < num_read_wqs; j++) - uobj_put_obj_read(wqs[j]); + rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_free: kfree(wqs_handles); kfree(wqs); @@ -3325,11 +3356,13 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) if (err) goto err_copy; - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); kfree(flow_attr); if (cmd.flow_attr.num_of_specs) kfree(kern_flow_attr); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: if (!qp->device->ops.destroy_flow(flow_id)) atomic_dec(&qp->usecnt); @@ -3338,7 +3371,8 @@ err_free: err_free_flow_attr: kfree(flow_attr); err_put: - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_uobj: uobj_alloc_abort(uobj, attrs); err_free_attr: @@ -3423,7 +3457,6 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, attr.attr.max_sge = cmd->max_sge; attr.attr.srq_limit = cmd->srq_limit; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); srq = rdma_zalloc_drv_obj(ib_dev, ib_srq); @@ -3435,7 +3468,7 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, srq->device = pd->device; srq->pd = pd; srq->srq_type = cmd->srq_type; - srq->uobject = &obj->uevent.uobject; + srq->uobject = obj; srq->event_handler = attr.event_handler; srq->srq_context = attr.srq_context; @@ -3474,10 +3507,12 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, uobj_put_read(xrcd_uobj); if (ib_srq_has_cq(cmd->srq_type)) - uobj_put_obj_read(attr.ext.cq); + rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); uobj_put_obj_read(pd); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_copy: ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs)); @@ -3490,7 +3525,8 @@ err_put: err_put_cq: if (ib_srq_has_cq(cmd->srq_type)) - uobj_put_obj_read(attr.ext.cq); + rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_put_xrcd: if (cmd->srq_type == IB_SRQT_XRC) { @@ -3558,7 +3594,8 @@ static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs) ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask, &attrs->driver_udata); - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -3581,7 +3618,8 @@ static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs) ret = ib_query_srq(srq, &attr); - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) return ret; @@ -3706,8 +3744,8 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs) ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period); - uobj_put_obj_read(cq); - + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c index 269938f59d3f..538affbc517e 100644 --- a/drivers/infiniband/core/uverbs_ioctl.c +++ b/drivers/infiniband/core/uverbs_ioctl.c @@ -220,24 +220,17 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle, return ret; } -static int uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi, - struct uverbs_objs_arr_attr *attr, - bool commit, struct uverbs_attr_bundle *attrs) +static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi, + struct uverbs_objs_arr_attr *attr, + bool commit, + struct uverbs_attr_bundle *attrs) { const struct uverbs_attr_spec *spec = &attr_uapi->spec; - int current_ret; - int ret = 0; size_t i; - for (i = 0; i != attr->len; i++) { - current_ret = uverbs_finalize_object(attr->uobjects[i], - spec->u2.objs_arr.access, - commit, attrs); - if (!ret) - ret = current_ret; - } - - return ret; + for (i = 0; i != attr->len; i++) + uverbs_finalize_object(attr->uobjects[i], + spec->u2.objs_arr.access, commit, attrs); } static int uverbs_process_attr(struct bundle_priv *pbundle, @@ -495,26 +488,22 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle, return ret; } -static int bundle_destroy(struct bundle_priv *pbundle, bool commit) +static void bundle_destroy(struct bundle_priv *pbundle, bool commit) { unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len; struct bundle_alloc_head *memblock; unsigned int i; - int ret = 0; /* fast path for simple uobjects */ i = -1; while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len, i + 1)) < key_bitmap_len) { struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; - int current_ret; - current_ret = uverbs_finalize_object( + uverbs_finalize_object( attr->obj_attr.uobject, attr->obj_attr.attr_elm->spec.u.obj.access, commit, &pbundle->bundle); - if (!ret) - ret = current_ret; } i = -1; @@ -523,7 +512,6 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit) struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; const struct uverbs_api_attr *attr_uapi; void __rcu **slot; - int current_ret; slot = uapi_get_attr_for_method( pbundle, @@ -534,11 +522,8 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit) attr_uapi = rcu_dereference_protected(*slot, true); if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) { - current_ret = uverbs_free_idrs_array( - attr_uapi, &attr->objs_arr_attr, commit, - &pbundle->bundle); - if (!ret) - ret = current_ret; + uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr, + commit, &pbundle->bundle); } } @@ -548,8 +533,6 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit) memblock = memblock->next; kvfree(tmp); } - - return ret; } static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, @@ -562,7 +545,6 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, struct bundle_priv *pbundle; struct bundle_priv onstack; void __rcu **slot; - int destroy_ret; int ret; if (unlikely(hdr->driver_id != uapi->driver_id)) @@ -610,10 +592,7 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize)); ret = ib_uverbs_run_method(pbundle, hdr->num_attrs); - destroy_ret = bundle_destroy(pbundle, ret == 0); - if (unlikely(destroy_ret && !ret)) - return destroy_ret; - + bundle_destroy(pbundle, ret == 0); return ret; } diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 970d8e31dd65..2d4083bf4a04 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -125,17 +125,8 @@ static void ib_uverbs_release_dev(struct device *device) kfree(dev); } -static void ib_uverbs_release_async_event_file(struct kref *ref) -{ - struct ib_uverbs_async_event_file *file = - container_of(ref, struct ib_uverbs_async_event_file, ref); - - kfree(file); -} - -void ib_uverbs_release_ucq(struct ib_uverbs_file *file, - struct ib_uverbs_completion_event_file *ev_file, - struct ib_ucq_object *uobj) +void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file, + struct ib_ucq_object *uobj) { struct ib_uverbs_event *evt, *tmp; @@ -150,25 +141,24 @@ void ib_uverbs_release_ucq(struct ib_uverbs_file *file, uverbs_uobject_put(&ev_file->uobj); } - spin_lock_irq(&file->async_file->ev_queue.lock); - list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) { - list_del(&evt->list); - kfree(evt); - } - spin_unlock_irq(&file->async_file->ev_queue.lock); + ib_uverbs_release_uevent(&uobj->uevent); } -void ib_uverbs_release_uevent(struct ib_uverbs_file *file, - struct ib_uevent_object *uobj) +void ib_uverbs_release_uevent(struct ib_uevent_object *uobj) { + struct ib_uverbs_async_event_file *async_file = + READ_ONCE(uobj->uobject.ufile->async_file); struct ib_uverbs_event *evt, *tmp; - spin_lock_irq(&file->async_file->ev_queue.lock); + if (!async_file) + return; + + spin_lock_irq(&async_file->ev_queue.lock); list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) { list_del(&evt->list); kfree(evt); } - spin_unlock_irq(&file->async_file->ev_queue.lock); + spin_unlock_irq(&async_file->ev_queue.lock); } void ib_uverbs_detach_umcast(struct ib_qp *qp, @@ -208,8 +198,7 @@ void ib_uverbs_release_file(struct kref *ref) ib_uverbs_comp_dev(file->device); if (file->async_file) - kref_put(&file->async_file->ref, - ib_uverbs_release_async_event_file); + uverbs_uobject_put(&file->async_file->uobj); put_device(&file->device->dev); if (file->disassociate_page) @@ -220,7 +209,6 @@ void ib_uverbs_release_file(struct kref *ref) } static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, - struct ib_uverbs_file *uverbs_file, struct file *filp, char __user *buf, size_t count, loff_t *pos, size_t eventsz) @@ -238,19 +226,16 @@ static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, if (wait_event_interruptible(ev_queue->poll_wait, (!list_empty(&ev_queue->event_list) || - /* The barriers built into wait_event_interruptible() - * and wake_up() guarentee this will see the null set - * without using RCU - */ - !uverbs_file->device->ib_dev))) + ev_queue->is_closed))) return -ERESTARTSYS; + spin_lock_irq(&ev_queue->lock); + /* If device was disassociated and no event exists set an error */ - if (list_empty(&ev_queue->event_list) && - !uverbs_file->device->ib_dev) + if (list_empty(&ev_queue->event_list) && ev_queue->is_closed) { + spin_unlock_irq(&ev_queue->lock); return -EIO; - - spin_lock_irq(&ev_queue->lock); + } } event = list_entry(ev_queue->event_list.next, struct ib_uverbs_event, list); @@ -285,8 +270,7 @@ static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf, { struct ib_uverbs_async_event_file *file = filp->private_data; - return ib_uverbs_event_read(&file->ev_queue, file->uverbs_file, filp, - buf, count, pos, + return ib_uverbs_event_read(&file->ev_queue, filp, buf, count, pos, sizeof(struct ib_uverbs_async_event_desc)); } @@ -296,9 +280,8 @@ static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf, struct ib_uverbs_completion_event_file *comp_ev_file = filp->private_data; - return ib_uverbs_event_read(&comp_ev_file->ev_queue, - comp_ev_file->uobj.ufile, filp, - buf, count, pos, + return ib_uverbs_event_read(&comp_ev_file->ev_queue, filp, buf, count, + pos, sizeof(struct ib_uverbs_comp_event_desc)); } @@ -321,7 +304,9 @@ static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue, static __poll_t ib_uverbs_async_event_poll(struct file *filp, struct poll_table_struct *wait) { - return ib_uverbs_event_poll(filp->private_data, filp, wait); + struct ib_uverbs_async_event_file *file = filp->private_data; + + return ib_uverbs_event_poll(&file->ev_queue, filp, wait); } static __poll_t ib_uverbs_comp_event_poll(struct file *filp, @@ -335,9 +320,9 @@ static __poll_t ib_uverbs_comp_event_poll(struct file *filp, static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on) { - struct ib_uverbs_event_queue *ev_queue = filp->private_data; + struct ib_uverbs_async_event_file *file = filp->private_data; - return fasync_helper(fd, filp, on, &ev_queue->async_queue); + return fasync_helper(fd, filp, on, &file->ev_queue.async_queue); } static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on) @@ -348,70 +333,20 @@ static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on) return fasync_helper(fd, filp, on, &comp_ev_file->ev_queue.async_queue); } -static int ib_uverbs_async_event_close(struct inode *inode, struct file *filp) -{ - struct ib_uverbs_async_event_file *file = filp->private_data; - struct ib_uverbs_file *uverbs_file = file->uverbs_file; - struct ib_uverbs_event *entry, *tmp; - int closed_already = 0; - - mutex_lock(&uverbs_file->device->lists_mutex); - spin_lock_irq(&file->ev_queue.lock); - closed_already = file->ev_queue.is_closed; - file->ev_queue.is_closed = 1; - list_for_each_entry_safe(entry, tmp, &file->ev_queue.event_list, list) { - if (entry->counter) - list_del(&entry->obj_list); - kfree(entry); - } - spin_unlock_irq(&file->ev_queue.lock); - if (!closed_already) { - list_del(&file->list); - ib_unregister_event_handler(&uverbs_file->event_handler); - } - mutex_unlock(&uverbs_file->device->lists_mutex); - - kref_put(&uverbs_file->ref, ib_uverbs_release_file); - kref_put(&file->ref, ib_uverbs_release_async_event_file); - - return 0; -} - -static int ib_uverbs_comp_event_close(struct inode *inode, struct file *filp) -{ - struct ib_uobject *uobj = filp->private_data; - struct ib_uverbs_completion_event_file *file = container_of( - uobj, struct ib_uverbs_completion_event_file, uobj); - struct ib_uverbs_event *entry, *tmp; - - spin_lock_irq(&file->ev_queue.lock); - list_for_each_entry_safe(entry, tmp, &file->ev_queue.event_list, list) { - if (entry->counter) - list_del(&entry->obj_list); - kfree(entry); - } - file->ev_queue.is_closed = 1; - spin_unlock_irq(&file->ev_queue.lock); - - uverbs_close_fd(filp); - - return 0; -} - const struct file_operations uverbs_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_comp_event_read, .poll = ib_uverbs_comp_event_poll, - .release = ib_uverbs_comp_event_close, + .release = uverbs_uobject_fd_release, .fasync = ib_uverbs_comp_event_fasync, .llseek = no_llseek, }; -static const struct file_operations uverbs_async_event_fops = { +const struct file_operations uverbs_async_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_async_event_read, .poll = ib_uverbs_async_event_poll, - .release = ib_uverbs_async_event_close, + .release = uverbs_uobject_fd_release, .fasync = ib_uverbs_async_event_fasync, .llseek = no_llseek, }; @@ -438,9 +373,9 @@ void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) return; } - uobj = container_of(cq->uobject, struct ib_ucq_object, uobject); + uobj = cq->uobject; - entry->desc.comp.cq_handle = cq->uobject->user_handle; + entry->desc.comp.cq_handle = cq->uobject->uevent.uobject.user_handle; entry->counter = &uobj->comp_events_reported; list_add_tail(&entry->list, &ev_queue->event_list); @@ -451,102 +386,82 @@ void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) kill_fasync(&ev_queue->async_queue, SIGIO, POLL_IN); } -static void ib_uverbs_async_handler(struct ib_uverbs_file *file, - __u64 element, __u64 event, - struct list_head *obj_list, - u32 *counter) +static void +ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file, + __u64 element, __u64 event, struct list_head *obj_list, + u32 *counter) { struct ib_uverbs_event *entry; unsigned long flags; - spin_lock_irqsave(&file->async_file->ev_queue.lock, flags); - if (file->async_file->ev_queue.is_closed) { - spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); + if (!async_file) + return; + + spin_lock_irqsave(&async_file->ev_queue.lock, flags); + if (async_file->ev_queue.is_closed) { + spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); return; } entry = kmalloc(sizeof(*entry), GFP_ATOMIC); if (!entry) { - spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); + spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); return; } - entry->desc.async.element = element; + entry->desc.async.element = element; entry->desc.async.event_type = event; - entry->desc.async.reserved = 0; - entry->counter = counter; + entry->desc.async.reserved = 0; + entry->counter = counter; - list_add_tail(&entry->list, &file->async_file->ev_queue.event_list); + list_add_tail(&entry->list, &async_file->ev_queue.event_list); if (obj_list) list_add_tail(&entry->obj_list, obj_list); - spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); + spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); - wake_up_interruptible(&file->async_file->ev_queue.poll_wait); - kill_fasync(&file->async_file->ev_queue.async_queue, SIGIO, POLL_IN); + wake_up_interruptible(&async_file->ev_queue.poll_wait); + kill_fasync(&async_file->ev_queue.async_queue, SIGIO, POLL_IN); } -void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) +static void uverbs_uobj_event(struct ib_uevent_object *eobj, + struct ib_event *event) { - struct ib_ucq_object *uobj = container_of(event->element.cq->uobject, - struct ib_ucq_object, uobject); + ib_uverbs_async_handler(READ_ONCE(eobj->uobject.ufile->async_file), + eobj->uobject.user_handle, event->event, + &eobj->event_list, &eobj->events_reported); +} - ib_uverbs_async_handler(uobj->uobject.ufile, uobj->uobject.user_handle, - event->event, &uobj->async_list, - &uobj->async_events_reported); +void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) +{ + uverbs_uobj_event(&event->element.cq->uobject->uevent, event); } void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj; - /* for XRC target qp's, check that qp is live */ if (!event->element.qp->uobject) return; - uobj = container_of(event->element.qp->uobject, - struct ib_uevent_object, uobject); - - ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, - event->event, &uobj->event_list, - &uobj->events_reported); + uverbs_uobj_event(&event->element.qp->uobject->uevent, event); } void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj = container_of(event->element.wq->uobject, - struct ib_uevent_object, uobject); - - ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, - event->event, &uobj->event_list, - &uobj->events_reported); + uverbs_uobj_event(&event->element.wq->uobject->uevent, event); } void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj; - - uobj = container_of(event->element.srq->uobject, - struct ib_uevent_object, uobject); - - ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, - event->event, &uobj->event_list, - &uobj->events_reported); -} - -void ib_uverbs_event_handler(struct ib_event_handler *handler, - struct ib_event *event) -{ - struct ib_uverbs_file *file = - container_of(handler, struct ib_uverbs_file, event_handler); - - ib_uverbs_async_handler(file, event->element.port_num, event->event, - NULL, NULL); + uverbs_uobj_event(&event->element.srq->uobject->uevent, event); } -void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file) +static void ib_uverbs_event_handler(struct ib_event_handler *handler, + struct ib_event *event) { - kref_put(&file->async_file->ref, ib_uverbs_release_async_event_file); - file->async_file = NULL; + ib_uverbs_async_handler( + container_of(handler, struct ib_uverbs_async_event_file, + event_handler), + event->element.port_num, event->event, NULL, NULL); } void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue) @@ -558,45 +473,26 @@ void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue) ev_queue->async_queue = NULL; } -struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file, - struct ib_device *ib_dev) +void ib_uverbs_init_async_event_file( + struct ib_uverbs_async_event_file *async_file) { - struct ib_uverbs_async_event_file *ev_file; - struct file *filp; - - ev_file = kzalloc(sizeof(*ev_file), GFP_KERNEL); - if (!ev_file) - return ERR_PTR(-ENOMEM); - - ib_uverbs_init_event_queue(&ev_file->ev_queue); - ev_file->uverbs_file = uverbs_file; - kref_get(&ev_file->uverbs_file->ref); - kref_init(&ev_file->ref); - filp = anon_inode_getfile("[infinibandevent]", &uverbs_async_event_fops, - ev_file, O_RDONLY); - if (IS_ERR(filp)) - goto err_put_refs; - - mutex_lock(&uverbs_file->device->lists_mutex); - list_add_tail(&ev_file->list, - &uverbs_file->device->uverbs_events_file_list); - mutex_unlock(&uverbs_file->device->lists_mutex); - - WARN_ON(uverbs_file->async_file); - uverbs_file->async_file = ev_file; - kref_get(&uverbs_file->async_file->ref); - INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler, - ib_dev, - ib_uverbs_event_handler); - ib_register_event_handler(&uverbs_file->event_handler); - /* At that point async file stuff was fully set */ + struct ib_uverbs_file *uverbs_file = async_file->uobj.ufile; + struct ib_device *ib_dev = async_file->uobj.context->device; + + ib_uverbs_init_event_queue(&async_file->ev_queue); - return filp; + /* The first async_event_file becomes the default one for the file. */ + mutex_lock(&uverbs_file->ucontext_lock); + if (!uverbs_file->async_file) { + /* Pairs with the put in ib_uverbs_release_file */ + uverbs_uobject_get(&async_file->uobj); + smp_store_release(&uverbs_file->async_file, async_file); + } + mutex_unlock(&uverbs_file->ucontext_lock); -err_put_refs: - kref_put(&ev_file->uverbs_file->ref, ib_uverbs_release_file); - kref_put(&ev_file->ref, ib_uverbs_release_async_event_file); - return filp; + INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev, + ib_uverbs_event_handler); + ib_register_event_handler(&async_file->event_handler); } static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr, @@ -1225,7 +1121,6 @@ static void ib_uverbs_add_one(struct ib_device *device) mutex_init(&uverbs_dev->xrcd_tree_mutex); mutex_init(&uverbs_dev->lists_mutex); INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list); - INIT_LIST_HEAD(&uverbs_dev->uverbs_events_file_list); rcu_assign_pointer(uverbs_dev->ib_dev, device); uverbs_dev->num_comp_vectors = device->num_comp_vectors; @@ -1270,14 +1165,9 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, struct ib_device *ib_dev) { struct ib_uverbs_file *file; - struct ib_uverbs_async_event_file *event_file; - struct ib_event event; /* Pending running commands to terminate */ uverbs_disassociate_api_pre(uverbs_dev); - event.event = IB_EVENT_DEVICE_FATAL; - event.element.port_num = 0; - event.device = ib_dev; mutex_lock(&uverbs_dev->lists_mutex); while (!list_empty(&uverbs_dev->uverbs_file_list)) { @@ -1293,31 +1183,14 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, */ mutex_unlock(&uverbs_dev->lists_mutex); - ib_uverbs_event_handler(&file->event_handler, &event); + ib_uverbs_async_handler(READ_ONCE(file->async_file), 0, + IB_EVENT_DEVICE_FATAL, NULL, NULL); + uverbs_destroy_ufile_hw(file, RDMA_REMOVE_DRIVER_REMOVE); kref_put(&file->ref, ib_uverbs_release_file); mutex_lock(&uverbs_dev->lists_mutex); } - - while (!list_empty(&uverbs_dev->uverbs_events_file_list)) { - event_file = list_first_entry(&uverbs_dev-> - uverbs_events_file_list, - struct ib_uverbs_async_event_file, - list); - spin_lock_irq(&event_file->ev_queue.lock); - event_file->ev_queue.is_closed = 1; - spin_unlock_irq(&event_file->ev_queue.lock); - - list_del(&event_file->list); - ib_unregister_event_handler( - &event_file->uverbs_file->event_handler); - event_file->uverbs_file->event_handler.device = - NULL; - - wake_up_interruptible(&event_file->ev_queue.poll_wait); - kill_fasync(&event_file->ev_queue.async_queue, SIGIO, POLL_IN); - } mutex_unlock(&uverbs_dev->lists_mutex); uverbs_disassociate_api(uverbs_dev->uapi); diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index 35b2e2c640cc..994d8744b246 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -105,7 +105,7 @@ static int uverbs_free_qp(struct ib_uobject *uobject, if (uqp->uxrcd) atomic_dec(&uqp->uxrcd->refcnt); - ib_uverbs_release_uevent(attrs->ufile, &uqp->uevent); + ib_uverbs_release_uevent(&uqp->uevent); return ret; } @@ -138,7 +138,7 @@ static int uverbs_free_wq(struct ib_uobject *uobject, if (ib_is_destroy_retryable(ret, why, uobject)) return ret; - ib_uverbs_release_uevent(attrs->ufile, &uwq->uevent); + ib_uverbs_release_uevent(&uwq->uevent); return ret; } @@ -163,7 +163,7 @@ static int uverbs_free_srq(struct ib_uobject *uobject, atomic_dec(&us->uxrcd->refcnt); } - ib_uverbs_release_uevent(attrs->ufile, uevent); + ib_uverbs_release_uevent(uevent); return ret; } @@ -202,24 +202,40 @@ static int uverbs_free_pd(struct ib_uobject *uobject, return 0; } -static int uverbs_hot_unplug_completion_event_file(struct ib_uobject *uobj, - enum rdma_remove_reason why) +void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue) { - struct ib_uverbs_completion_event_file *comp_event_file = - container_of(uobj, struct ib_uverbs_completion_event_file, - uobj); - struct ib_uverbs_event_queue *event_queue = &comp_event_file->ev_queue; + struct ib_uverbs_event *entry, *tmp; spin_lock_irq(&event_queue->lock); + /* + * The user must ensure that no new items are added to the event_list + * once is_closed is set. + */ event_queue->is_closed = 1; spin_unlock_irq(&event_queue->lock); + wake_up_interruptible(&event_queue->poll_wait); + kill_fasync(&event_queue->async_queue, SIGIO, POLL_IN); - if (why == RDMA_REMOVE_DRIVER_REMOVE) { - wake_up_interruptible(&event_queue->poll_wait); - kill_fasync(&event_queue->async_queue, SIGIO, POLL_IN); + spin_lock_irq(&event_queue->lock); + list_for_each_entry_safe(entry, tmp, &event_queue->event_list, list) { + if (entry->counter) + list_del(&entry->obj_list); + kfree(entry); } + spin_unlock_irq(&event_queue->lock); +} + +static int +uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + struct ib_uverbs_completion_event_file *file = + container_of(uobj, struct ib_uverbs_completion_event_file, + uobj); + + ib_uverbs_free_event_queue(&file->ev_queue); return 0; -}; +} int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) { @@ -230,7 +246,7 @@ EXPORT_SYMBOL(uverbs_destroy_def_handler); DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_COMP_CHANNEL, UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file), - uverbs_hot_unplug_completion_event_file, + uverbs_completion_event_file_destroy_uobj, &uverbs_event_fops, "[infinibandevent]", O_RDONLY)); diff --git a/drivers/infiniband/core/uverbs_std_types_async_fd.c b/drivers/infiniband/core/uverbs_std_types_async_fd.c new file mode 100644 index 000000000000..82ec0806b34b --- /dev/null +++ b/drivers/infiniband/core/uverbs_std_types_async_fd.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. + */ + +#include <rdma/uverbs_std_types.h> +#include <rdma/uverbs_ioctl.h> +#include "rdma_core.h" +#include "uverbs.h" + +static int UVERBS_HANDLER(UVERBS_METHOD_ASYNC_EVENT_ALLOC)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = + uverbs_attr_get_uobject(attrs, UVERBS_METHOD_ASYNC_EVENT_ALLOC); + + ib_uverbs_init_async_event_file( + container_of(uobj, struct ib_uverbs_async_event_file, uobj)); + return 0; +} + +static int uverbs_async_event_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + struct ib_uverbs_async_event_file *event_file = + container_of(uobj, struct ib_uverbs_async_event_file, uobj); + + ib_unregister_event_handler(&event_file->event_handler); + ib_uverbs_free_event_queue(&event_file->ev_queue); + return 0; +} + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_ASYNC_EVENT_ALLOC, + UVERBS_ATTR_FD(UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE, + UVERBS_OBJECT_ASYNC_EVENT, + UVERBS_ACCESS_NEW, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_OBJECT( + UVERBS_OBJECT_ASYNC_EVENT, + UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file), + uverbs_async_event_destroy_uobj, + &uverbs_async_event_fops, + "[infinibandevent]", + O_RDONLY), + &UVERBS_METHOD(UVERBS_METHOD_ASYNC_EVENT_ALLOC)); + +const struct uapi_definition uverbs_def_obj_async_fd[] = { + UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_ASYNC_EVENT), + {} +}; diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index e39fe6a8aac4..da4110a0eea2 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -41,7 +41,7 @@ static int uverbs_free_cq(struct ib_uobject *uobject, struct ib_cq *cq = uobject->object; struct ib_uverbs_event_queue *ev_queue = cq->cq_context; struct ib_ucq_object *ucq = - container_of(uobject, struct ib_ucq_object, uobject); + container_of(uobject, struct ib_ucq_object, uevent.uobject); int ret; ret = ib_destroy_cq_user(cq, &attrs->driver_udata); @@ -49,7 +49,6 @@ static int uverbs_free_cq(struct ib_uobject *uobject, return ret; ib_uverbs_release_ucq( - attrs->ufile, ev_queue ? container_of(ev_queue, struct ib_uverbs_completion_event_file, ev_queue) : @@ -63,7 +62,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( { struct ib_ucq_object *obj = container_of( uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE), - typeof(*obj), uobject); + typeof(*obj), uevent.uobject); struct ib_device *ib_dev = attrs->context->device; int ret; u64 user_handle; @@ -106,10 +105,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( goto err_event_file; } - obj->comp_events_reported = 0; - obj->async_events_reported = 0; INIT_LIST_HEAD(&obj->comp_list); - INIT_LIST_HEAD(&obj->async_list); + INIT_LIST_HEAD(&obj->uevent.event_list); cq = rdma_zalloc_drv_obj(ib_dev, ib_cq); if (!cq) { @@ -118,7 +115,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( } cq->device = ib_dev; - cq->uobject = &obj->uobject; + cq->uobject = obj; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; @@ -129,8 +126,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( if (ret) goto err_free; - obj->uobject.object = cq; - obj->uobject.user_handle = user_handle; + obj->uevent.uobject.object = cq; + obj->uevent.uobject.user_handle = user_handle; rdma_restrack_uadd(&cq->res); ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &cq->cqe, @@ -182,10 +179,10 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)( struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_CQ_HANDLE); struct ib_ucq_object *obj = - container_of(uobj, struct ib_ucq_object, uobject); + container_of(uobj, struct ib_ucq_object, uevent.uobject); struct ib_uverbs_destroy_cq_resp resp = { .comp_events_reported = obj->comp_events_reported, - .async_events_reported = obj->async_events_reported + .async_events_reported = obj->uevent.events_reported }; return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_CQ_RESP, &resp, diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index 2a3f2f01028d..ae4a59d6f9b1 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -200,6 +200,43 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_PORT)( &resp, sizeof(resp)); } +static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( + struct uverbs_attr_bundle *attrs) +{ + u32 num_comp = attrs->ufile->device->num_comp_vectors; + u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS; + int ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, + &num_comp, sizeof(num_comp)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, + &core_support, sizeof(core_support)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + + ret = ib_alloc_ucontext(attrs); + if (ret) + return ret; + ret = ib_init_ucontext(attrs); + if (ret) { + kfree(attrs->context); + attrs->context = NULL; + return ret; + } + return 0; +} + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_GET_CONTEXT, + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, + UVERBS_ATTR_TYPE(u32), UA_OPTIONAL), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, + UVERBS_ATTR_TYPE(u64), UA_OPTIONAL), + UVERBS_ATTR_UHW()); + DECLARE_UVERBS_NAMED_METHOD( UVERBS_METHOD_INFO_HANDLES, /* Also includes any device specific object ids */ @@ -220,6 +257,7 @@ DECLARE_UVERBS_NAMED_METHOD( UA_MANDATORY)); DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, + &UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE), &UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES), &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT)); diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index 00c547887132..3f121ac31e0a 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -195,9 +195,9 @@ static int uapi_merge_obj_tree(struct uverbs_api *uapi, * disassociation, and the FD types require the driver to use * struct file_operations.owner to prevent the driver module * code from unloading while the file is open. This provides - * enough safety that uverbs_close_fd() will continue to work. - * Drivers using FD are responsible to handle disassociation of - * the device on their own. + * enough safety that uverbs_uobject_fd_release() will + * continue to work. Drivers using FD are responsible to + * handle disassociation of the device on their own. */ if (WARN_ON(is_driver && obj->type_attrs->type_class != &uverbs_idr_class && @@ -626,6 +626,7 @@ void uverbs_destroy_api(struct uverbs_api *uapi) } static const struct uapi_definition uverbs_core_api[] = { + UAPI_DEF_CHAIN(uverbs_def_obj_async_fd), UAPI_DEF_CHAIN(uverbs_def_obj_counters), UAPI_DEF_CHAIN(uverbs_def_obj_cq), UAPI_DEF_CHAIN(uverbs_def_obj_device), diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index d33bdc9b01cd..3ebae3b65c28 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -52,6 +52,9 @@ #include <rdma/rw.h> #include "core_priv.h" +#include <trace/events/rdma_core.h> + +#include <trace/events/rdma_core.h> static int ib_resolve_eth_dmac(struct ib_device *device, struct rdma_ah_attr *ah_attr); @@ -1053,11 +1056,11 @@ static void __ib_shared_qp_event_handler(struct ib_event *event, void *context) struct ib_qp *qp = context; unsigned long flags; - spin_lock_irqsave(&qp->device->event_handler_lock, flags); + spin_lock_irqsave(&qp->device->qp_open_list_lock, flags); list_for_each_entry(event->element.qp, &qp->open_list, open_list) if (event->element.qp->event_handler) event->element.qp->event_handler(event, event->element.qp->qp_context); - spin_unlock_irqrestore(&qp->device->event_handler_lock, flags); + spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags); } static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) @@ -1094,9 +1097,9 @@ static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp, qp->qp_num = real_qp->qp_num; qp->qp_type = real_qp->qp_type; - spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); + spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags); list_add(&qp->open_list, &real_qp->open_list); - spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); + spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags); return qp; } @@ -1824,9 +1827,9 @@ int ib_close_qp(struct ib_qp *qp) if (real_qp == qp) return -EINVAL; - spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); + spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags); list_del(&qp->open_list); - spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); + spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags); atomic_dec(&real_qp->usecnt); if (qp->qp_sec) @@ -2038,6 +2041,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata) struct ib_sig_attrs *sig_attrs = mr->sig_attrs; int ret; + trace_mr_dereg(mr); rdma_restrack_del(&mr->res); ret = mr->device->ops.dereg_mr(mr, udata); if (!ret) { @@ -2069,11 +2073,16 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type, { struct ib_mr *mr; - if (!pd->device->ops.alloc_mr) - return ERR_PTR(-EOPNOTSUPP); + if (!pd->device->ops.alloc_mr) { + mr = ERR_PTR(-EOPNOTSUPP); + goto out; + } - if (WARN_ON_ONCE(mr_type == IB_MR_TYPE_INTEGRITY)) - return ERR_PTR(-EINVAL); + if (mr_type == IB_MR_TYPE_INTEGRITY) { + WARN_ON_ONCE(1); + mr = ERR_PTR(-EINVAL); + goto out; + } mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata); if (!IS_ERR(mr)) { @@ -2089,6 +2098,8 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type, mr->sig_attrs = NULL; } +out: + trace_mr_alloc(pd, mr_type, max_num_sg, mr); return mr; } EXPORT_SYMBOL(ib_alloc_mr_user); @@ -2113,21 +2124,27 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd, struct ib_sig_attrs *sig_attrs; if (!pd->device->ops.alloc_mr_integrity || - !pd->device->ops.map_mr_sg_pi) - return ERR_PTR(-EOPNOTSUPP); + !pd->device->ops.map_mr_sg_pi) { + mr = ERR_PTR(-EOPNOTSUPP); + goto out; + } - if (!max_num_meta_sg) - return ERR_PTR(-EINVAL); + if (!max_num_meta_sg) { + mr = ERR_PTR(-EINVAL); + goto out; + } sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL); - if (!sig_attrs) - return ERR_PTR(-ENOMEM); + if (!sig_attrs) { + mr = ERR_PTR(-ENOMEM); + goto out; + } mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg, max_num_meta_sg); if (IS_ERR(mr)) { kfree(sig_attrs); - return mr; + goto out; } mr->device = pd->device; @@ -2141,6 +2158,8 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd, mr->type = IB_MR_TYPE_INTEGRITY; mr->sig_attrs = sig_attrs; +out: + trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr); return mr; } EXPORT_SYMBOL(ib_alloc_mr_integrity); @@ -2785,6 +2804,7 @@ void ib_drain_sq(struct ib_qp *qp) qp->device->ops.drain_sq(qp); else __ib_drain_sq(qp); + trace_cq_drain_complete(qp->send_cq); } EXPORT_SYMBOL(ib_drain_sq); @@ -2813,6 +2833,7 @@ void ib_drain_rq(struct ib_qp *qp) qp->device->ops.drain_rq(qp); else __ib_drain_rq(qp); + trace_cq_drain_complete(qp->recv_cq); } EXPORT_SYMBOL(ib_drain_rq); diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index e7e8a0f49464..793c97251588 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -677,7 +677,7 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid); - ibdev->num_comp_vectors = 1; + ibdev->num_comp_vectors = rdev->num_msix - 1; ibdev->dev.parent = &rdev->en_dev->pdev->dev; ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index e96bcb16bd2b..74b787a90660 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -160,10 +160,16 @@ struct efa_admin_create_qp_resp { /* Common Admin Queue completion descriptor */ struct efa_admin_acq_common_desc acq_common_desc; - /* Opaque handle to be used for consequent operations on the QP */ + /* + * Opaque handle to be used for consequent admin operations on the + * QP + */ u32 qp_handle; - /* QP number in the given EFA virtual device */ + /* + * QP number in the given EFA virtual device. Least-significant bits + * (as needed according to max_qp) carry unique QP ID + */ u16 qp_num; /* MBZ */ @@ -286,6 +292,7 @@ struct efa_admin_create_ah_cmd { /* PD number */ u16 pd; + /* MBZ */ u16 reserved; }; @@ -296,6 +303,7 @@ struct efa_admin_create_ah_resp { /* Target interface address handle (opaque) */ u16 ah; + /* MBZ */ u16 reserved; }; @@ -372,6 +380,7 @@ struct efa_admin_reg_mr_cmd { */ u8 permissions; + /* MBZ */ u16 reserved16_w5; /* number of pages in PBL (redundant, could be calculated) */ @@ -419,20 +428,20 @@ struct efa_admin_create_cq_cmd { struct efa_admin_aq_common_desc aq_common_desc; /* - * 4:0 : reserved5 + * 4:0 : reserved5 - MBZ * 5 : interrupt_mode_enabled - if set, cq operates * in interrupt mode (i.e. CQ events and MSI-X are * generated), otherwise - polling * 6 : virt - If set, ring base address is virtual * (IOVA returned by MR registration) - * 7 : reserved6 + * 7 : reserved6 - MBZ */ u8 cq_caps_1; /* * 4:0 : cq_entry_size_words - size of CQ entry in * 32-bit words, valid values: 4, 8. - * 7:5 : reserved7 + * 7:5 : reserved7 - MBZ */ u8 cq_caps_2; @@ -478,6 +487,7 @@ struct efa_admin_destroy_cq_cmd { u16 cq_idx; + /* MBZ */ u16 reserved1; }; @@ -530,7 +540,7 @@ struct efa_admin_get_set_feature_common_desc { /* * 1:0 : select - 0x1 - current value; 0x3 - default * value - * 7:3 : reserved3 + * 7:3 : reserved3 - MBZ */ u8 flags; @@ -557,10 +567,10 @@ struct efa_admin_feature_device_attr_desc { /* Bar used for SQ and RQ doorbells */ u16 db_bar; - /* Indicates how many bits are used physical address access */ + /* Indicates how many bits are used on physical address access */ u8 phys_addr_width; - /* Indicates how many bits are used virtual address access */ + /* Indicates how many bits are used on virtual address access */ u8 virt_addr_width; /* @@ -578,27 +588,28 @@ struct efa_admin_feature_queue_attr_desc { /* The maximum number of queue pairs supported */ u32 max_qp; + /* Maximum number of WQEs per Send Queue */ u32 max_sq_depth; - /* max send wr used in inline-buf */ + /* Maximum size of data that can be sent inline in a Send WQE */ u32 inline_buf_size; + /* Maximum number of buffer descriptors per Recv Queue */ u32 max_rq_depth; /* The maximum number of completion queues supported per VF */ u32 max_cq; + /* Maximum number of CQEs per Completion Queue */ u32 max_cq_depth; /* Number of sub-CQs to be created for each CQ */ u16 sub_cqs_per_cq; + /* MBZ */ u16 reserved; - /* - * Maximum number of SGEs (buffs) allowed for a single send work - * queue element (WQE) - */ + /* Maximum number of SGEs (buffers) allowed for a single send WQE */ u16 max_wr_send_sges; /* Maximum number of SGEs allowed for a single recv WQE */ diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 4822f5fa12be..ec5545870554 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -387,8 +387,7 @@ static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle) return efa_com_destroy_qp(&dev->edev, ¶ms); } -static void efa_qp_user_mmap_entries_remove(struct efa_ucontext *uctx, - struct efa_qp *qp) +static void efa_qp_user_mmap_entries_remove(struct efa_qp *qp) { rdma_user_mmap_entry_remove(qp->rq_mmap_entry); rdma_user_mmap_entry_remove(qp->rq_db_mmap_entry); @@ -398,8 +397,6 @@ static void efa_qp_user_mmap_entries_remove(struct efa_ucontext *uctx, int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { - struct efa_ucontext *ucontext = rdma_udata_to_drv_context(udata, - struct efa_ucontext, ibucontext); struct efa_dev *dev = to_edev(ibqp->pd->device); struct efa_qp *qp = to_eqp(ibqp); int err; @@ -418,7 +415,7 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) DMA_TO_DEVICE); } - efa_qp_user_mmap_entries_remove(ucontext, qp); + efa_qp_user_mmap_entries_remove(qp); kfree(qp); return 0; } @@ -510,7 +507,7 @@ static int qp_mmap_entries_setup(struct efa_qp *qp, return 0; err_remove_mmap: - efa_qp_user_mmap_entries_remove(ucontext, qp); + efa_qp_user_mmap_entries_remove(qp); return -ENOMEM; } @@ -719,7 +716,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd, return &qp->ibqp; err_remove_mmap_entries: - efa_qp_user_mmap_entries_remove(ucontext, qp); + efa_qp_user_mmap_entries_remove(qp); err_destroy_qp: efa_destroy_qp_handle(dev, create_qp_resp.qp_handle); err_free_mapped: @@ -1370,6 +1367,7 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, IB_ACCESS_LOCAL_WRITE | (is_rdma_read_cap(dev) ? IB_ACCESS_REMOTE_READ : 0); + access_flags &= ~IB_ACCESS_OPTIONAL; if (access_flags & ~supp_access_flags) { ibdev_dbg(&dev->ibdev, "Unsupported access flags[%#x], supported[%#x]\n", @@ -1608,13 +1606,12 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL; } - if (err) { + if (err) ibdev_dbg( &dev->ibdev, "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", entry->address, rdma_entry->npages * PAGE_SIZE, entry->mmap_flag, err); - } rdma_user_mmap_entry_put(rdma_entry); return err; diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 9b1fb84a3d45..e0b1238d31df 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -1685,6 +1685,14 @@ static u64 access_sw_pio_drain(const struct cntr_entry *entry, return dd->verbs_dev.n_piodrain; } +static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry, + void *context, int vl, int mode, u64 data) +{ + struct hfi1_devdata *dd = context; + + return dd->ctx0_seq_drop; +} + static u64 access_sw_vtx_wait(const struct cntr_entry *entry, void *context, int vl, int mode, u64 data) { @@ -4106,6 +4114,7 @@ def_access_ibp_counter(rc_crwaits); static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = { [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH), [C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH), +[C_RX_SHORT_ERR] = RXE32_DEV_CNTR_ELEM(RxShrErr, RCV_SHORT_ERR_CNT, CNTR_SYNTH), [C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH), [C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH), [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT, @@ -4249,6 +4258,8 @@ static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = { access_sw_cpu_intr), [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL, access_sw_cpu_rcv_limit), +[C_SW_CTX0_SEQ_DROP] = CNTR_ELEM("SeqDrop0", 0, 0, CNTR_NORMAL, + access_sw_ctx0_seq_drop), [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL, access_sw_vtx_wait), [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL, @@ -6862,7 +6873,7 @@ static void rxe_kernel_unfreeze(struct hfi1_devdata *dd) } rcvmask = HFI1_RCVCTRL_CTXT_ENB; /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */ - rcvmask |= rcd->rcvhdrtail_kvaddr ? + rcvmask |= hfi1_rcvhdrtail_kvaddr(rcd) ? HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS; hfi1_rcvctrl(dd, rcvmask, rcd); hfi1_rcd_put(rcd); @@ -8394,20 +8405,62 @@ void force_recv_intr(struct hfi1_ctxtdata *rcd) static inline int check_packet_present(struct hfi1_ctxtdata *rcd) { u32 tail; - int present; - if (!rcd->rcvhdrtail_kvaddr) - present = (rcd->seq_cnt == - rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd)))); - else /* is RDMA rtail */ - present = (rcd->head != get_rcvhdrtail(rcd)); - - if (present) + if (hfi1_packet_present(rcd)) return 1; /* fall back to a CSR read, correct indpendent of DMA_RTAIL */ tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); - return rcd->head != tail; + return hfi1_rcd_head(rcd) != tail; +} + +/** + * Common code for receive contexts interrupt handlers. + * Update traces, increment kernel IRQ counter and + * setup ASPM when needed. + */ +static void receive_interrupt_common(struct hfi1_ctxtdata *rcd) +{ + struct hfi1_devdata *dd = rcd->dd; + + trace_hfi1_receive_interrupt(dd, rcd); + this_cpu_inc(*dd->int_counter); + aspm_ctx_disable(rcd); +} + +/** + * __hfi1_rcd_eoi_intr() - Make HW issue receive interrupt + * when there are packets present in the queue. When calling + * with interrupts enabled please use hfi1_rcd_eoi_intr. + * + * @rcd: valid receive context + */ +static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) +{ + clear_recv_intr(rcd); + if (check_packet_present(rcd)) + force_recv_intr(rcd); +} + +/** + * hfi1_rcd_eoi_intr() - End of Interrupt processing action + * + * @rcd: Ptr to hfi1_ctxtdata of receive context + * + * Hold IRQs so we can safely clear the interrupt and + * recheck for a packet that may have arrived after the previous + * check and the interrupt clear. If a packet arrived, force another + * interrupt. This routine can be called at the end of receive packet + * processing in interrupt service routines, interrupt service thread + * and softirqs + */ +static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) +{ + unsigned long flags; + + local_irq_save(flags); + __hfi1_rcd_eoi_intr(rcd); + local_irq_restore(flags); } /* @@ -8421,13 +8474,9 @@ static inline int check_packet_present(struct hfi1_ctxtdata *rcd) irqreturn_t receive_context_interrupt(int irq, void *data) { struct hfi1_ctxtdata *rcd = data; - struct hfi1_devdata *dd = rcd->dd; int disposition; - int present; - trace_hfi1_receive_interrupt(dd, rcd); - this_cpu_inc(*dd->int_counter); - aspm_ctx_disable(rcd); + receive_interrupt_common(rcd); /* receive interrupt remains blocked while processing packets */ disposition = rcd->do_interrupt(rcd, 0); @@ -8440,17 +8489,7 @@ irqreturn_t receive_context_interrupt(int irq, void *data) if (disposition == RCV_PKT_LIMIT) return IRQ_WAKE_THREAD; - /* - * The packet processor detected no more packets. Clear the receive - * interrupt and recheck for a packet packet that may have arrived - * after the previous check and interrupt clear. If a packet arrived, - * force another interrupt. - */ - clear_recv_intr(rcd); - present = check_packet_present(rcd); - if (present) - force_recv_intr(rcd); - + __hfi1_rcd_eoi_intr(rcd); return IRQ_HANDLED; } @@ -8461,24 +8500,11 @@ irqreturn_t receive_context_interrupt(int irq, void *data) irqreturn_t receive_context_thread(int irq, void *data) { struct hfi1_ctxtdata *rcd = data; - int present; /* receive interrupt is still blocked from the IRQ handler */ (void)rcd->do_interrupt(rcd, 1); - /* - * The packet processor will only return if it detected no more - * packets. Hold IRQs here so we can safely clear the interrupt and - * recheck for a packet that may have arrived after the previous - * check and the interrupt clear. If a packet arrived, force another - * interrupt. - */ - local_irq_disable(); - clear_recv_intr(rcd); - present = check_packet_present(rcd); - if (present) - force_recv_intr(rcd); - local_irq_enable(); + hfi1_rcd_eoi_intr(rcd); return IRQ_HANDLED; } @@ -10049,7 +10075,7 @@ u32 lrh_max_header_bytes(struct hfi1_devdata *dd) * the first kernel context would have been allocated by now so * we are guaranteed a valid value. */ - return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2; + return (get_hdrqentsize(dd->rcd[0]) - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2; } /* @@ -10094,7 +10120,7 @@ static void set_send_length(struct hfi1_pportdata *ppd) thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50), sc_mtu_to_threshold(dd->vld[i].sc, dd->vld[i].mtu, - dd->rcd[0]->rcvhdrqentsize)); + get_hdrqentsize(dd->rcd[0]))); for (j = 0; j < INIT_SC_PER_VL; j++) sc_set_cr_threshold( pio_select_send_context_vl(dd, j, i), @@ -11821,7 +11847,7 @@ u32 hdrqempty(struct hfi1_ctxtdata *rcd) head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT; - if (rcd->rcvhdrtail_kvaddr) + if (hfi1_rcvhdrtail_kvaddr(rcd)) tail = get_rcvhdrtail(rcd); else tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); @@ -11865,6 +11891,84 @@ static u32 encoded_size(u32 size) return 0x1; /* if invalid, go with the minimum size */ } +/** + * encode_rcv_header_entry_size - return chip specific encoding for size + * @size: size in dwords + * + * Convert a receive header entry size that to the encoding used in the CSR. + * + * Return a zero if the given size is invalid, otherwise the encoding. + */ +u8 encode_rcv_header_entry_size(u8 size) +{ + /* there are only 3 valid receive header entry sizes */ + if (size == 2) + return 1; + if (size == 16) + return 2; + if (size == 32) + return 4; + return 0; /* invalid */ +} + +/** + * hfi1_validate_rcvhdrcnt - validate hdrcnt + * @dd: the device data + * @thecnt: the header count + */ +int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt) +{ + if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) { + dd_dev_err(dd, "Receive header queue count too small\n"); + return -EINVAL; + } + + if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) { + dd_dev_err(dd, + "Receive header queue count cannot be greater than %u\n", + HFI1_MAX_HDRQ_EGRBUF_CNT); + return -EINVAL; + } + + if (thecnt % HDRQ_INCREMENT) { + dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n", + thecnt, HDRQ_INCREMENT); + return -EINVAL; + } + + return 0; +} + +/** + * set_hdrq_regs - set header queue registers for context + * @dd: the device data + * @ctxt: the context + * @entsize: the dword entry size + * @hdrcnt: the number of header entries + */ +void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt) +{ + u64 reg; + + reg = (((u64)hdrcnt >> HDRQ_SIZE_SHIFT) & RCV_HDR_CNT_CNT_MASK) << + RCV_HDR_CNT_CNT_SHIFT; + write_kctxt_csr(dd, ctxt, RCV_HDR_CNT, reg); + reg = ((u64)encode_rcv_header_entry_size(entsize) & + RCV_HDR_ENT_SIZE_ENT_SIZE_MASK) << + RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT; + write_kctxt_csr(dd, ctxt, RCV_HDR_ENT_SIZE, reg); + reg = ((u64)DEFAULT_RCVHDRSIZE & RCV_HDR_SIZE_HDR_SIZE_MASK) << + RCV_HDR_SIZE_HDR_SIZE_SHIFT; + write_kctxt_csr(dd, ctxt, RCV_HDR_SIZE, reg); + + /* + * Program dummy tail address for every receive context + * before enabling any receive context + */ + write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, + dd->rcvhdrtail_dummy_dma); +} + void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, struct hfi1_ctxtdata *rcd) { @@ -11886,13 +11990,13 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, /* reset the tail and hdr addresses, and sequence count */ write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR, rcd->rcvhdrq_dma); - if (rcd->rcvhdrtail_kvaddr) + if (hfi1_rcvhdrtail_kvaddr(rcd)) write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, rcd->rcvhdrqtailaddr_dma); - rcd->seq_cnt = 1; + hfi1_set_seq_cnt(rcd, 1); /* reset the cached receive header queue head value */ - rcd->head = 0; + hfi1_set_rcd_head(rcd, 0); /* * Zero the receive header queue so we don't get false @@ -11972,7 +12076,7 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, IS_RCVAVAIL_START + rcd->ctxt, false); rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK; } - if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && rcd->rcvhdrtail_kvaddr) + if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && hfi1_rcvhdrtail_kvaddr(rcd)) rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; if (op & HFI1_RCVCTRL_TAILUPD_DIS) { /* See comment on RcvCtxtCtrl.TailUpd above */ diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h index 4ca5ac8d7e9e..725509261016 100644 --- a/drivers/infiniband/hw/hfi1/chip.h +++ b/drivers/infiniband/hw/hfi1/chip.h @@ -358,6 +358,8 @@ #define MAX_EAGER_BUFFER (256 * 1024) #define MAX_EAGER_BUFFER_TOTAL (64 * (1 << 20)) /* max per ctxt 64MB */ #define MAX_EXPECTED_BUFFER (2048 * 1024) +#define HFI1_MIN_HDRQ_EGRBUF_CNT 32 +#define HFI1_MAX_HDRQ_EGRBUF_CNT 16352 /* * Receive expected base and count and eager base and count increment - @@ -699,6 +701,10 @@ static inline u32 chip_rcv_array_count(struct hfi1_devdata *dd) return read_csr(dd, RCV_ARRAY_CNT); } +u8 encode_rcv_header_entry_size(u8 size); +int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt); +void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt); + u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl, u32 dw_len); @@ -859,6 +865,7 @@ static inline int idx_from_vl(int vl) enum { C_RCV_OVF = 0, C_RX_LEN_ERR, + C_RX_SHORT_ERR, C_RX_ICRC_ERR, C_RX_EBP, C_RX_TID_FULL, @@ -926,6 +933,7 @@ enum { C_DC_PG_STS_TX_MBE_CNT, C_SW_CPU_INTR, C_SW_CPU_RCV_LIM, + C_SW_CTX0_SEQ_DROP, C_SW_VTX_WAIT, C_SW_PIO_WAIT, C_SW_PIO_DRAIN, diff --git a/drivers/infiniband/hw/hfi1/chip_registers.h b/drivers/infiniband/hw/hfi1/chip_registers.h index ab3589d17aee..fb3ec9bff7a2 100644 --- a/drivers/infiniband/hw/hfi1/chip_registers.h +++ b/drivers/infiniband/hw/hfi1/chip_registers.h @@ -381,6 +381,7 @@ #define DC_LCB_STS_LINK_TRANSFER_ACTIVE (DC_LCB_CSRS + 0x000000000468) #define DC_LCB_STS_ROUND_TRIP_LTP_CNT (DC_LCB_CSRS + 0x0000000004B0) #define RCV_LENGTH_ERR_CNT 0 +#define RCV_SHORT_ERR_CNT 2 #define RCV_ICRC_ERR_CNT 6 #define RCV_EBP_CNT 9 #define RCV_BUF_OVFL_CNT 10 diff --git a/drivers/infiniband/hw/hfi1/common.h b/drivers/infiniband/hw/hfi1/common.h index d47da7b0438f..40a1ff0c8a8e 100644 --- a/drivers/infiniband/hw/hfi1/common.h +++ b/drivers/infiniband/hw/hfi1/common.h @@ -323,6 +323,9 @@ struct diag_pkt { /* RHF receive type error - bypass packet errors */ #define RHF_RTE_BYPASS_NO_ERR 0x0 +/* MAX RcvSEQ */ +#define RHF_MAX_SEQ 13 + /* IB - LRH header constants */ #define HFI1_LRH_GRH 0x0003 /* 1. word of IB LRH - next header: GRH */ #define HFI1_LRH_BTH 0x0002 /* 1. word of IB LRH - next header: BTH */ diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c index d268bf9c42ee..4633a0ce1a8c 100644 --- a/drivers/infiniband/hw/hfi1/debugfs.c +++ b/drivers/infiniband/hw/hfi1/debugfs.c @@ -379,7 +379,7 @@ static void *_rcds_seq_next(struct seq_file *s, void *v, loff_t *pos) struct hfi1_devdata *dd = dd_from_dev(ibd); ++*pos; - if (!dd->rcd || *pos >= dd->n_krcv_queues) + if (!dd->rcd || *pos >= dd->num_rcv_contexts) return NULL; return pos; } diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c index 01aa1f132f55..049d15befe58 100644 --- a/drivers/infiniband/hw/hfi1/driver.c +++ b/drivers/infiniband/hw/hfi1/driver.c @@ -411,14 +411,14 @@ drop: static inline void init_packet(struct hfi1_ctxtdata *rcd, struct hfi1_packet *packet) { - packet->rsize = rcd->rcvhdrqentsize; /* words */ - packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */ + packet->rsize = get_hdrqentsize(rcd); /* words */ + packet->maxcnt = get_hdrq_cnt(rcd) * packet->rsize; /* words */ packet->rcd = rcd; packet->updegr = 0; packet->etail = -1; packet->rhf_addr = get_rhf_addr(rcd); packet->rhf = rhf_to_cpu(packet->rhf_addr); - packet->rhqoff = rcd->head; + packet->rhqoff = hfi1_rcd_head(rcd); packet->numpkt = 0; } @@ -551,22 +551,22 @@ static inline void init_ps_mdata(struct ps_mdata *mdata, mdata->maxcnt = packet->maxcnt; mdata->ps_head = packet->rhqoff; - if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { + if (get_dma_rtail_setting(rcd)) { mdata->ps_tail = get_rcvhdrtail(rcd); if (rcd->ctxt == HFI1_CTRL_CTXT) - mdata->ps_seq = rcd->seq_cnt; + mdata->ps_seq = hfi1_seq_cnt(rcd); else mdata->ps_seq = 0; /* not used with DMA_RTAIL */ } else { mdata->ps_tail = 0; /* used only with DMA_RTAIL*/ - mdata->ps_seq = rcd->seq_cnt; + mdata->ps_seq = hfi1_seq_cnt(rcd); } } static inline int ps_done(struct ps_mdata *mdata, u64 rhf, struct hfi1_ctxtdata *rcd) { - if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) + if (get_dma_rtail_setting(rcd)) return mdata->ps_head == mdata->ps_tail; return mdata->ps_seq != rhf_rcv_seq(rhf); } @@ -592,11 +592,9 @@ static inline void update_ps_mdata(struct ps_mdata *mdata, mdata->ps_head = 0; /* Control context must do seq counting */ - if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || - (rcd->ctxt == HFI1_CTRL_CTXT)) { - if (++mdata->ps_seq > 13) - mdata->ps_seq = 1; - } + if (!get_dma_rtail_setting(rcd) || + rcd->ctxt == HFI1_CTRL_CTXT) + mdata->ps_seq = hfi1_seq_incr_wrap(mdata->ps_seq); } /* @@ -734,6 +732,7 @@ static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) { int ret; + packet->rcd->dd->ctx0_seq_drop++; /* Set up for the next packet */ packet->rhqoff += packet->rsize; if (packet->rhqoff >= packet->maxcnt) @@ -769,7 +768,7 @@ static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) * The +2 is the size of the RHF. */ prefetch_range(packet->ebuf, - packet->tlen - ((packet->rcd->rcvhdrqentsize - + packet->tlen - ((get_hdrqentsize(packet->rcd) - (rhf_hdrq_offset(packet->rhf) + 2)) * 4)); } @@ -823,7 +822,7 @@ static inline void finish_packet(struct hfi1_packet *packet) * The only thing we need to do is a final update and call for an * interrupt */ - update_usrhead(packet->rcd, packet->rcd->head, packet->updegr, + update_usrhead(packet->rcd, hfi1_rcd_head(packet->rcd), packet->updegr, packet->etail, rcv_intr_dynamic, packet->numpkt); } @@ -832,13 +831,11 @@ static inline void finish_packet(struct hfi1_packet *packet) */ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) { - u32 seq; int last = RCV_PKT_OK; struct hfi1_packet packet; init_packet(rcd, &packet); - seq = rhf_rcv_seq(packet.rhf); - if (seq != rcd->seq_cnt) { + if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) { last = RCV_PKT_DONE; goto bail; } @@ -847,15 +844,12 @@ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) while (last == RCV_PKT_OK) { last = process_rcv_packet(&packet, thread); - seq = rhf_rcv_seq(packet.rhf); - if (++rcd->seq_cnt > 13) - rcd->seq_cnt = 1; - if (seq != rcd->seq_cnt) + if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) last = RCV_PKT_DONE; process_rcv_update(last, &packet); } process_rcv_qp_work(&packet); - rcd->head = packet.rhqoff; + hfi1_set_rcd_head(rcd, packet.rhqoff); bail: finish_packet(&packet); return last; @@ -884,15 +878,14 @@ int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) process_rcv_update(last, &packet); } process_rcv_qp_work(&packet); - rcd->head = packet.rhqoff; + hfi1_set_rcd_head(rcd, packet.rhqoff); bail: finish_packet(&packet); return last; } -static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt) +static void set_all_fastpath(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) { - struct hfi1_ctxtdata *rcd; u16 i; /* @@ -900,50 +893,17 @@ static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt) * interrupt handler only for that context. Otherwise, switch * interrupt handler for all statically allocated kernel contexts. */ - if (ctxt >= dd->first_dyn_alloc_ctxt) { - rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); - if (rcd) { - rcd->do_interrupt = - &handle_receive_interrupt_nodma_rtail; - hfi1_rcd_put(rcd); - } - return; - } - - for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { - rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) - rcd->do_interrupt = - &handle_receive_interrupt_nodma_rtail; + if (rcd->ctxt >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic) { + hfi1_rcd_get(rcd); + hfi1_set_fast(rcd); hfi1_rcd_put(rcd); - } -} - -static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt) -{ - struct hfi1_ctxtdata *rcd; - u16 i; - - /* - * For dynamically allocated kernel contexts (like vnic) switch - * interrupt handler only for that context. Otherwise, switch - * interrupt handler for all statically allocated kernel contexts. - */ - if (ctxt >= dd->first_dyn_alloc_ctxt) { - rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); - if (rcd) { - rcd->do_interrupt = - &handle_receive_interrupt_dma_rtail; - hfi1_rcd_put(rcd); - } return; } - for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { + for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) { rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) - rcd->do_interrupt = - &handle_receive_interrupt_dma_rtail; + if (rcd && (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)) + hfi1_set_fast(rcd); hfi1_rcd_put(rcd); } } @@ -959,17 +919,14 @@ void set_all_slowpath(struct hfi1_devdata *dd) if (!rcd) continue; if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic) - rcd->do_interrupt = &handle_receive_interrupt; + rcd->do_interrupt = rcd->slow_handler; hfi1_rcd_put(rcd); } } -static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, - struct hfi1_packet *packet, - struct hfi1_devdata *dd) +static bool __set_armed_to_active(struct hfi1_packet *packet) { - struct work_struct *lsaw = &rcd->ppd->linkstate_active_work; u8 etype = rhf_rcv_type(packet->rhf); u8 sc = SC15_PACKET; @@ -984,19 +941,34 @@ static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, sc = hfi1_16B_get_sc(hdr); } if (sc != SC15_PACKET) { - int hwstate = driver_lstate(rcd->ppd); + int hwstate = driver_lstate(packet->rcd->ppd); + struct work_struct *lsaw = + &packet->rcd->ppd->linkstate_active_work; if (hwstate != IB_PORT_ACTIVE) { - dd_dev_info(dd, + dd_dev_info(packet->rcd->dd, "Unexpected link state %s\n", opa_lstate_name(hwstate)); - return 0; + return false; } - queue_work(rcd->ppd->link_wq, lsaw); - return 1; + queue_work(packet->rcd->ppd->link_wq, lsaw); + return true; } - return 0; + return false; +} + +/** + * armed to active - the fast path for armed to active + * @packet: the packet structure + * + * Return true if packet processing needs to bail. + */ +static bool set_armed_to_active(struct hfi1_packet *packet) +{ + if (likely(packet->rcd->ppd->host_link_state != HLS_UP_ARMED)) + return false; + return __set_armed_to_active(packet); } /* @@ -1019,10 +991,8 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) init_packet(rcd, &packet); - if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { - u32 seq = rhf_rcv_seq(packet.rhf); - - if (seq != rcd->seq_cnt) { + if (!get_dma_rtail_setting(rcd)) { + if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) { last = RCV_PKT_DONE; goto bail; } @@ -1039,22 +1009,15 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) * Control context can potentially receive an invalid * rhf. Drop such packets. */ - if (rcd->ctxt == HFI1_CTRL_CTXT) { - u32 seq = rhf_rcv_seq(packet.rhf); - - if (seq != rcd->seq_cnt) + if (rcd->ctxt == HFI1_CTRL_CTXT) + if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) skip_pkt = 1; - } } prescan_rxq(rcd, &packet); while (last == RCV_PKT_OK) { - if (unlikely(dd->do_drop && - atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) == - DROP_PACKET_ON)) { - dd->do_drop = 0; - + if (hfi1_need_drop(dd)) { /* On to the next packet */ packet.rhqoff += packet.rsize; packet.rhf_addr = (__le32 *)rcd->rcvhdrq + @@ -1066,26 +1029,14 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) last = skip_rcv_packet(&packet, thread); skip_pkt = 0; } else { - /* Auto activate link on non-SC15 packet receive */ - if (unlikely(rcd->ppd->host_link_state == - HLS_UP_ARMED) && - set_armed_to_active(rcd, &packet, dd)) + if (set_armed_to_active(&packet)) goto bail; last = process_rcv_packet(&packet, thread); } - if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { - u32 seq = rhf_rcv_seq(packet.rhf); - - if (++rcd->seq_cnt > 13) - rcd->seq_cnt = 1; - if (seq != rcd->seq_cnt) + if (!get_dma_rtail_setting(rcd)) { + if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) last = RCV_PKT_DONE; - if (needset) { - dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n"); - set_nodma_rtail(dd, rcd->ctxt); - needset = 0; - } } else { if (packet.rhqoff == hdrqtail) last = RCV_PKT_DONE; @@ -1094,27 +1045,24 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) * rhf. Drop such packets. */ if (rcd->ctxt == HFI1_CTRL_CTXT) { - u32 seq = rhf_rcv_seq(packet.rhf); + bool lseq; - if (++rcd->seq_cnt > 13) - rcd->seq_cnt = 1; - if (!last && (seq != rcd->seq_cnt)) + lseq = hfi1_seq_incr(rcd, + rhf_rcv_seq(packet.rhf)); + if (!last && lseq) skip_pkt = 1; } - - if (needset) { - dd_dev_info(dd, - "Switching to DMA_RTAIL\n"); - set_dma_rtail(dd, rcd->ctxt); - needset = 0; - } } + if (needset) { + needset = false; + set_all_fastpath(dd, rcd); + } process_rcv_update(last, &packet); } process_rcv_qp_work(&packet); - rcd->head = packet.rhqoff; + hfi1_set_rcd_head(rcd, packet.rhqoff); bail: /* @@ -1606,23 +1554,22 @@ void handle_eflags(struct hfi1_packet *packet) * The following functions are called by the interrupt handler. They are type * specific handlers for each packet type. */ -static int process_receive_ib(struct hfi1_packet *packet) +static void process_receive_ib(struct hfi1_packet *packet) { if (hfi1_setup_9B_packet(packet)) - return RHF_RCV_CONTINUE; + return; if (unlikely(hfi1_dbg_should_fault_rx(packet))) - return RHF_RCV_CONTINUE; + return; trace_hfi1_rcvhdr(packet); if (unlikely(rhf_err_flags(packet->rhf))) { handle_eflags(packet); - return RHF_RCV_CONTINUE; + return; } hfi1_ib_rcv(packet); - return RHF_RCV_CONTINUE; } static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet) @@ -1638,23 +1585,23 @@ static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet) return false; } -static int process_receive_bypass(struct hfi1_packet *packet) +static void process_receive_bypass(struct hfi1_packet *packet) { struct hfi1_devdata *dd = packet->rcd->dd; if (hfi1_is_vnic_packet(packet)) { hfi1_vnic_bypass_rcv(packet); - return RHF_RCV_CONTINUE; + return; } if (hfi1_setup_bypass_packet(packet)) - return RHF_RCV_CONTINUE; + return; trace_hfi1_rcvhdr(packet); if (unlikely(rhf_err_flags(packet->rhf))) { handle_eflags(packet); - return RHF_RCV_CONTINUE; + return; } if (hfi1_16B_get_l2(packet->hdr) == 0x2) { @@ -1677,17 +1624,16 @@ static int process_receive_bypass(struct hfi1_packet *packet) (OPA_EI_STATUS_SMASK | BAD_L2_ERR); } } - return RHF_RCV_CONTINUE; } -static int process_receive_error(struct hfi1_packet *packet) +static void process_receive_error(struct hfi1_packet *packet) { /* KHdrHCRCErr -- KDETH packet with a bad HCRC */ if (unlikely( hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR || packet->rhf & RHF_DC_ERR))) - return RHF_RCV_CONTINUE; + return; hfi1_setup_ib_header(packet); handle_eflags(packet); @@ -1695,32 +1641,29 @@ static int process_receive_error(struct hfi1_packet *packet) if (unlikely(rhf_err_flags(packet->rhf))) dd_dev_err(packet->rcd->dd, "Unhandled error packet received. Dropping.\n"); - - return RHF_RCV_CONTINUE; } -static int kdeth_process_expected(struct hfi1_packet *packet) +static void kdeth_process_expected(struct hfi1_packet *packet) { hfi1_setup_9B_packet(packet); if (unlikely(hfi1_dbg_should_fault_rx(packet))) - return RHF_RCV_CONTINUE; + return; if (unlikely(rhf_err_flags(packet->rhf))) { struct hfi1_ctxtdata *rcd = packet->rcd; if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) - return RHF_RCV_CONTINUE; + return; } hfi1_kdeth_expected_rcv(packet); - return RHF_RCV_CONTINUE; } -static int kdeth_process_eager(struct hfi1_packet *packet) +static void kdeth_process_eager(struct hfi1_packet *packet) { hfi1_setup_9B_packet(packet); if (unlikely(hfi1_dbg_should_fault_rx(packet))) - return RHF_RCV_CONTINUE; + return; trace_hfi1_rcvhdr(packet); if (unlikely(rhf_err_flags(packet->rhf))) { @@ -1728,37 +1671,41 @@ static int kdeth_process_eager(struct hfi1_packet *packet) show_eflags_errs(packet); if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) - return RHF_RCV_CONTINUE; + return; } hfi1_kdeth_eager_rcv(packet); - return RHF_RCV_CONTINUE; } -static int process_receive_invalid(struct hfi1_packet *packet) +static void process_receive_invalid(struct hfi1_packet *packet) { dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", rhf_rcv_type(packet->rhf)); - return RHF_RCV_CONTINUE; } +#define HFI1_RCVHDR_DUMP_MAX 5 + void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd) { struct hfi1_packet packet; struct ps_mdata mdata; + int i; - seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n", - rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize, - HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ? + seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s ctrl 0x%08llx status 0x%08llx, head %llu tail %llu sw head %u\n", + rcd->ctxt, get_hdrq_cnt(rcd), get_hdrqentsize(rcd), + get_dma_rtail_setting(rcd) ? "dma_rtail" : "nodma_rtail", + read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_CTRL), + read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_STATUS), read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) & RCV_HDR_HEAD_HEAD_MASK, - read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL)); + read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL), + rcd->head); init_packet(rcd, &packet); init_ps_mdata(&mdata, &packet); - while (1) { + for (i = 0; i < HFI1_RCVHDR_DUMP_MAX; i++) { __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + rcd->rhf_offset; struct ib_header *hdr; diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 7c5e3fb22413..bef6946861b2 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -505,12 +505,12 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) ret = -EINVAL; goto done; } - if ((flags & VM_WRITE) || !uctxt->rcvhdrtail_kvaddr) { + if ((flags & VM_WRITE) || !hfi1_rcvhdrtail_kvaddr(uctxt)) { ret = -EPERM; goto done; } memlen = PAGE_SIZE; - memvirt = (void *)uctxt->rcvhdrtail_kvaddr; + memvirt = (void *)hfi1_rcvhdrtail_kvaddr(uctxt); flags &= ~VM_MAYWRITE; break; case SUBCTXT_UREGS: @@ -1090,7 +1090,7 @@ static void user_init(struct hfi1_ctxtdata *uctxt) * don't have to wait to be sure the DMA update has happened * (chip resets head/tail to 0 on transition to enable). */ - if (uctxt->rcvhdrtail_kvaddr) + if (hfi1_rcvhdrtail_kvaddr(uctxt)) clear_rcvhdrtail(uctxt); /* Setup J_KEY before enabling the context */ @@ -1154,8 +1154,8 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len) cinfo.send_ctxt = uctxt->sc->hw_context; cinfo.egrtids = uctxt->egrbufs.alloced; - cinfo.rcvhdrq_cnt = uctxt->rcvhdrq_cnt; - cinfo.rcvhdrq_entsize = uctxt->rcvhdrqentsize << 2; + cinfo.rcvhdrq_cnt = get_hdrq_cnt(uctxt); + cinfo.rcvhdrq_entsize = get_hdrqentsize(uctxt) << 2; cinfo.sdma_ring_size = fd->cq->nentries; cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size; @@ -1543,7 +1543,7 @@ static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt, * always resets it's tail register back to 0 on a * transition from disabled to enabled. */ - if (uctxt->rcvhdrtail_kvaddr) + if (hfi1_rcvhdrtail_kvaddr(uctxt)) clear_rcvhdrtail(uctxt); rcvctrl_op = HFI1_RCVCTRL_CTXT_ENB; } else { diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h index fc10d65fc3e1..6365e8ffed9d 100644 --- a/drivers/infiniband/hw/hfi1/hfi.h +++ b/drivers/infiniband/hw/hfi1/hfi.h @@ -197,7 +197,9 @@ struct exp_tid_set { u32 count; }; -typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet); +struct hfi1_ctxtdata; +typedef int (*intr_handler)(struct hfi1_ctxtdata *rcd, int data); +typedef void (*rhf_rcv_function_ptr)(struct hfi1_packet *packet); struct tid_queue { struct list_head queue_head; @@ -226,7 +228,11 @@ struct hfi1_ctxtdata { * be valid. Worst case is we process an extra interrupt and up to 64 * packets with the wrong interrupt handler. */ - int (*do_interrupt)(struct hfi1_ctxtdata *rcd, int threaded); + intr_handler do_interrupt; + /** fast handler after autoactive */ + intr_handler fast_handler; + /** slow handler */ + intr_handler slow_handler; /* verbs rx_stats per rcd */ struct hfi1_opcode_stats_perctx *opstats; /* clear interrupt mask */ @@ -1153,6 +1159,8 @@ struct hfi1_devdata { char *boardname; /* human readable board info */ + u64 ctx0_seq_drop; + /* reset value */ u64 z_int_counter; u64 z_rcv_limit; @@ -1310,7 +1318,7 @@ struct hfi1_devdata { struct err_info_constraint err_info_xmit_constraint; atomic_t drop_packet; - u8 do_drop; + bool do_drop; u8 err_info_uncorrectable; u8 err_info_fmconfig; @@ -1507,12 +1515,148 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, #define RCV_PKT_LIMIT 0x1 /* stop, hit limit, start thread */ #define RCV_PKT_DONE 0x2 /* stop, no more packets detected */ +/** + * hfi1_rcd_head - add accessor for rcd head + * @rcd: the context + */ +static inline u32 hfi1_rcd_head(struct hfi1_ctxtdata *rcd) +{ + return rcd->head; +} + +/** + * hfi1_set_rcd_head - add accessor for rcd head + * @rcd: the context + * @head: the new head + */ +static inline void hfi1_set_rcd_head(struct hfi1_ctxtdata *rcd, u32 head) +{ + rcd->head = head; +} + /* calculate the current RHF address */ static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd) { return (__le32 *)rcd->rcvhdrq + rcd->head + rcd->rhf_offset; } +/* return DMA_RTAIL configuration */ +static inline bool get_dma_rtail_setting(struct hfi1_ctxtdata *rcd) +{ + return !!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL); +} + +/** + * hfi1_seq_incr_wrap - wrapping increment for sequence + * @seq: the current sequence number + * + * Returns: the incremented seq + */ +static inline u8 hfi1_seq_incr_wrap(u8 seq) +{ + if (++seq > RHF_MAX_SEQ) + seq = 1; + return seq; +} + +/** + * hfi1_seq_cnt - return seq_cnt member + * @rcd: the receive context + * + * Return seq_cnt member + */ +static inline u8 hfi1_seq_cnt(struct hfi1_ctxtdata *rcd) +{ + return rcd->seq_cnt; +} + +/** + * hfi1_set_seq_cnt - return seq_cnt member + * @rcd: the receive context + * + * Return seq_cnt member + */ +static inline void hfi1_set_seq_cnt(struct hfi1_ctxtdata *rcd, u8 cnt) +{ + rcd->seq_cnt = cnt; +} + +/** + * last_rcv_seq - is last + * @rcd: the receive context + * @seq: sequence + * + * return true if last packet + */ +static inline bool last_rcv_seq(struct hfi1_ctxtdata *rcd, u32 seq) +{ + return seq != rcd->seq_cnt; +} + +/** + * rcd_seq_incr - increment context sequence number + * @rcd: the receive context + * @seq: the current sequence number + * + * Returns: true if the this was the last packet + */ +static inline bool hfi1_seq_incr(struct hfi1_ctxtdata *rcd, u32 seq) +{ + rcd->seq_cnt = hfi1_seq_incr_wrap(rcd->seq_cnt); + return last_rcv_seq(rcd, seq); +} + +/** + * get_hdrqentsize - return hdrq entry size + * @rcd: the receive context + */ +static inline u8 get_hdrqentsize(struct hfi1_ctxtdata *rcd) +{ + return rcd->rcvhdrqentsize; +} + +/** + * get_hdrq_cnt - return hdrq count + * @rcd: the receive context + */ +static inline u16 get_hdrq_cnt(struct hfi1_ctxtdata *rcd) +{ + return rcd->rcvhdrq_cnt; +} + +/** + * hfi1_is_slowpath - check if this context is slow path + * @rcd: the receive context + */ +static inline bool hfi1_is_slowpath(struct hfi1_ctxtdata *rcd) +{ + return rcd->do_interrupt == rcd->slow_handler; +} + +/** + * hfi1_is_fastpath - check if this context is fast path + * @rcd: the receive context + */ +static inline bool hfi1_is_fastpath(struct hfi1_ctxtdata *rcd) +{ + if (rcd->ctxt == HFI1_CTRL_CTXT) + return false; + + return rcd->do_interrupt == rcd->fast_handler; +} + +/** + * hfi1_set_fast - change to the fast handler + * @rcd: the receive context + */ +static inline void hfi1_set_fast(struct hfi1_ctxtdata *rcd) +{ + if (unlikely(!rcd)) + return; + if (unlikely(!hfi1_is_fastpath(rcd))) + rcd->do_interrupt = rcd->fast_handler; +} + int hfi1_reset_device(int); void receive_interrupt_work(struct work_struct *work); @@ -2015,9 +2159,21 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, void hfi1_release_user_pages(struct mm_struct *mm, struct page **p, size_t npages, bool dirty); +/** + * hfi1_rcvhdrtail_kvaddr - return tail kvaddr + * @rcd - the receive context + */ +static inline __le64 *hfi1_rcvhdrtail_kvaddr(const struct hfi1_ctxtdata *rcd) +{ + return (__le64 *)rcd->rcvhdrtail_kvaddr; +} + static inline void clear_rcvhdrtail(const struct hfi1_ctxtdata *rcd) { - *((u64 *)rcd->rcvhdrtail_kvaddr) = 0ULL; + u64 *kv = (u64 *)hfi1_rcvhdrtail_kvaddr(rcd); + + if (kv) + *kv = 0ULL; } static inline u32 get_rcvhdrtail(const struct hfi1_ctxtdata *rcd) @@ -2026,7 +2182,17 @@ static inline u32 get_rcvhdrtail(const struct hfi1_ctxtdata *rcd) * volatile because it's a DMA target from the chip, routine is * inlined, and don't want register caching or reordering. */ - return (u32)le64_to_cpu(*rcd->rcvhdrtail_kvaddr); + return (u32)le64_to_cpu(*hfi1_rcvhdrtail_kvaddr(rcd)); +} + +static inline bool hfi1_packet_present(struct hfi1_ctxtdata *rcd) +{ + if (likely(!rcd->rcvhdrtail_kvaddr)) { + u32 seq = rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))); + + return !last_rcv_seq(rcd, seq); + } + return hfi1_rcd_head(rcd) != get_rcvhdrtail(rcd); } /* @@ -2298,6 +2464,25 @@ static inline bool is_integrated(struct hfi1_devdata *dd) return dd->pcidev->device == PCI_DEVICE_ID_INTEL1; } +/** + * hfi1_need_drop - detect need for drop + * @dd: - the device + * + * In some cases, the first packet needs to be dropped. + * + * Return true is the current packet needs to be dropped and false otherwise. + */ +static inline bool hfi1_need_drop(struct hfi1_devdata *dd) +{ + if (unlikely(dd->do_drop && + atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) == + DROP_PACKET_ON)) { + dd->do_drop = false; + return true; + } + return false; +} + int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp); #define DD_DEV_ENTRY(dd) __string(dev, dev_name(&(dd)->pcidev->dev)) diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 26b792bb1027..e3acda7a0800 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -78,8 +78,6 @@ */ #define HFI1_MIN_USER_CTXT_BUFCNT 7 -#define HFI1_MIN_HDRQ_EGRBUF_CNT 2 -#define HFI1_MAX_HDRQ_EGRBUF_CNT 16352 #define HFI1_MIN_EAGER_BUFFER_SIZE (4 * 1024) /* 4KB */ #define HFI1_MAX_EAGER_BUFFER_SIZE (256 * 1024) /* 256KB */ @@ -122,8 +120,6 @@ unsigned int user_credit_return_threshold = 33; /* default is 33% */ module_param(user_credit_return_threshold, uint, S_IRUGO); MODULE_PARM_DESC(user_credit_return_threshold, "Credit return threshold for user send contexts, return when unreturned credits passes this many blocks (in percent of allocated blocks, 0 is off)"); -static inline u64 encode_rcv_header_entry_size(u16 size); - DEFINE_XARRAY_FLAGS(hfi1_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); static int hfi1_create_kctxt(struct hfi1_devdata *dd, @@ -154,7 +150,12 @@ static int hfi1_create_kctxt(struct hfi1_devdata *dd, /* Control context must use DMA_RTAIL */ if (rcd->ctxt == HFI1_CTRL_CTXT) rcd->flags |= HFI1_CAP_DMA_RTAIL; - rcd->seq_cnt = 1; + rcd->fast_handler = get_dma_rtail_setting(rcd) ? + handle_receive_interrupt_dma_rtail : + handle_receive_interrupt_nodma_rtail; + rcd->slow_handler = handle_receive_interrupt; + + hfi1_set_seq_cnt(rcd, 1); rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node); if (!rcd->sc) { @@ -511,23 +512,6 @@ void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd) } /* - * Convert a receive header entry size that to the encoding used in the CSR. - * - * Return a zero if the given size is invalid. - */ -static inline u64 encode_rcv_header_entry_size(u16 size) -{ - /* there are only 3 valid receive header entry sizes */ - if (size == 2) - return 1; - if (size == 16) - return 2; - else if (size == 32) - return 4; - return 0; /* invalid */ -} - -/* * Select the largest ccti value over all SLs to determine the intra- * packet gap for the link. * @@ -892,10 +876,10 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit) if (is_ax(dd)) { atomic_set(&dd->drop_packet, DROP_PACKET_ON); - dd->do_drop = 1; + dd->do_drop = true; } else { atomic_set(&dd->drop_packet, DROP_PACKET_OFF); - dd->do_drop = 0; + dd->do_drop = false; } /* make sure the link is not "up" */ @@ -1149,9 +1133,9 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) dma_free_coherent(&dd->pcidev->dev, rcvhdrq_size(rcd), rcd->rcvhdrq, rcd->rcvhdrq_dma); rcd->rcvhdrq = NULL; - if (rcd->rcvhdrtail_kvaddr) { + if (hfi1_rcvhdrtail_kvaddr(rcd)) { dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE, - (void *)rcd->rcvhdrtail_kvaddr, + (void *)hfi1_rcvhdrtail_kvaddr(rcd), rcd->rcvhdrqtailaddr_dma); rcd->rcvhdrtail_kvaddr = NULL; } @@ -1611,29 +1595,6 @@ static void postinit_cleanup(struct hfi1_devdata *dd) hfi1_free_devdata(dd); } -static int init_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt) -{ - if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) { - dd_dev_err(dd, "Receive header queue count too small\n"); - return -EINVAL; - } - - if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) { - dd_dev_err(dd, - "Receive header queue count cannot be greater than %u\n", - HFI1_MAX_HDRQ_EGRBUF_CNT); - return -EINVAL; - } - - if (thecnt % HDRQ_INCREMENT) { - dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n", - thecnt, HDRQ_INCREMENT); - return -EINVAL; - } - - return 0; -} - static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { int ret = 0, j, pidx, initfail; @@ -1661,7 +1622,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Validate some global module parameters */ - ret = init_validate_rcvhdrcnt(dd, rcvhdrcnt); + ret = hfi1_validate_rcvhdrcnt(dd, rcvhdrcnt); if (ret) goto bail; @@ -1842,7 +1803,6 @@ static void shutdown_one(struct pci_dev *pdev) int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) { unsigned amt; - u64 reg; if (!rcd->rcvhdrq) { gfp_t gfp_flags; @@ -1874,30 +1834,9 @@ int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) goto bail_free; } } - /* - * These values are per-context: - * RcvHdrCnt - * RcvHdrEntSize - * RcvHdrSize - */ - reg = ((u64)(rcd->rcvhdrq_cnt >> HDRQ_SIZE_SHIFT) - & RCV_HDR_CNT_CNT_MASK) - << RCV_HDR_CNT_CNT_SHIFT; - write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_CNT, reg); - reg = (encode_rcv_header_entry_size(rcd->rcvhdrqentsize) - & RCV_HDR_ENT_SIZE_ENT_SIZE_MASK) - << RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT; - write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_ENT_SIZE, reg); - reg = ((u64)DEFAULT_RCVHDRSIZE & RCV_HDR_SIZE_HDR_SIZE_MASK) - << RCV_HDR_SIZE_HDR_SIZE_SHIFT; - write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_SIZE, reg); - /* - * Program dummy tail address for every receive context - * before enabling any receive context - */ - write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_TAIL_ADDR, - dd->rcvhdrtail_dummy_dma); + set_hdrq_regs(rcd->dd, rcd->ctxt, rcd->rcvhdrqentsize, + rcd->rcvhdrq_cnt); return 0; diff --git a/drivers/infiniband/hw/hfi1/msix.c b/drivers/infiniband/hw/hfi1/msix.c index d920b165d696..db82db497b2c 100644 --- a/drivers/infiniband/hw/hfi1/msix.c +++ b/drivers/infiniband/hw/hfi1/msix.c @@ -115,13 +115,11 @@ int msix_initialize(struct hfi1_devdata *dd) */ static int msix_request_irq(struct hfi1_devdata *dd, void *arg, irq_handler_t handler, irq_handler_t thread, - u32 idx, enum irq_type type) + enum irq_type type, const char *name) { unsigned long nr; int irq; int ret; - const char *err_info; - char name[MAX_NAME_SIZE]; struct hfi1_msix_entry *me; /* Allocate an MSIx vector */ @@ -135,43 +133,15 @@ static int msix_request_irq(struct hfi1_devdata *dd, void *arg, if (nr == dd->msix_info.max_requested) return -ENOSPC; - /* Specific verification and determine the name */ - switch (type) { - case IRQ_GENERAL: - /* general interrupt must be MSIx vector 0 */ - if (nr) { - spin_lock(&dd->msix_info.msix_lock); - __clear_bit(nr, dd->msix_info.in_use_msix); - spin_unlock(&dd->msix_info.msix_lock); - dd_dev_err(dd, "Invalid index %lu for GENERAL IRQ\n", - nr); - return -EINVAL; - } - snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit); - err_info = "general"; - break; - case IRQ_SDMA: - snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d", - dd->unit, idx); - err_info = "sdma"; - break; - case IRQ_RCVCTXT: - snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d", - dd->unit, idx); - err_info = "receive context"; - break; - case IRQ_OTHER: - default: + if (type < IRQ_SDMA || type >= IRQ_OTHER) return -EINVAL; - } - name[sizeof(name) - 1] = 0; irq = pci_irq_vector(dd->pcidev, nr); ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name); if (ret) { dd_dev_err(dd, - "%s: request for IRQ %d failed, MSIx %d, err %d\n", - err_info, irq, idx, ret); + "%s: request for IRQ %d failed, MSIx %lu, err %d\n", + name, irq, nr, ret); spin_lock(&dd->msix_info.msix_lock); __clear_bit(nr, dd->msix_info.in_use_msix); spin_unlock(&dd->msix_info.msix_lock); @@ -195,17 +165,13 @@ static int msix_request_irq(struct hfi1_devdata *dd, void *arg, return nr; } -/** - * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs - * @rcd: valid rcd context - * - */ -int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) +static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd, + irq_handler_t handler, + irq_handler_t thread, + const char *name) { - int nr; - - nr = msix_request_irq(rcd->dd, rcd, receive_context_interrupt, - receive_context_thread, rcd->ctxt, IRQ_RCVCTXT); + int nr = msix_request_irq(rcd->dd, rcd, handler, thread, + IRQ_RCVCTXT, name); if (nr < 0) return nr; @@ -222,6 +188,22 @@ int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) } /** + * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs + * @rcd: valid rcd context + * + */ +int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) +{ + char name[MAX_NAME_SIZE]; + + snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d", + rcd->dd->unit, rcd->ctxt); + + return msix_request_rcd_irq_common(rcd, receive_context_interrupt, + receive_context_thread, name); +} + +/** * msix_request_smda_ira() - Helper for getting SDMA IRQ resources * @sde: valid sdma engine * @@ -229,9 +211,12 @@ int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) int msix_request_sdma_irq(struct sdma_engine *sde) { int nr; + char name[MAX_NAME_SIZE]; + snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d", + sde->dd->unit, sde->this_idx); nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL, - sde->this_idx, IRQ_SDMA); + IRQ_SDMA, name); if (nr < 0) return nr; sde->msix_intr = nr; @@ -241,6 +226,32 @@ int msix_request_sdma_irq(struct sdma_engine *sde) } /** + * msix_request_general_irq(void) - Helper for getting general IRQ + * resources + * @dd: valid device data + */ +int msix_request_general_irq(struct hfi1_devdata *dd) +{ + int nr; + char name[MAX_NAME_SIZE]; + + snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit); + nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL, + name); + if (nr < 0) + return nr; + + /* general interrupt must be MSIx vector 0 */ + if (nr) { + msix_free_irq(dd, (u8)nr); + dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr); + return -EINVAL; + } + + return 0; +} + +/** * enable_sdma_src() - Helper to enable SDMA IRQ srcs * @dd: valid devdata structure * @i: index of SDMA engine @@ -265,10 +276,9 @@ static void enable_sdma_srcs(struct hfi1_devdata *dd, int i) int msix_request_irqs(struct hfi1_devdata *dd) { int i; - int ret; + int ret = msix_request_general_irq(dd); - ret = msix_request_irq(dd, dd, general_interrupt, NULL, 0, IRQ_GENERAL); - if (ret < 0) + if (ret) return ret; for (i = 0; i < dd->num_sdma; i++) { diff --git a/drivers/infiniband/hw/hfi1/msix.h b/drivers/infiniband/hw/hfi1/msix.h index a514881632a4..1a02ab7971c8 100644 --- a/drivers/infiniband/hw/hfi1/msix.h +++ b/drivers/infiniband/hw/hfi1/msix.h @@ -54,6 +54,7 @@ int msix_initialize(struct hfi1_devdata *dd); int msix_request_irqs(struct hfi1_devdata *dd); void msix_clean_up_interrupts(struct hfi1_devdata *dd); +int msix_request_general_irq(struct hfi1_devdata *dd); int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd); int msix_request_sdma_irq(struct sdma_engine *sde); void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr); diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index 1a3c647675a7..f1734e5e9ac4 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -2599,7 +2599,7 @@ static noinline int rc_rcv_error(struct ib_other_headers *ohdr, void *data, * to be sent before sending this one. */ e = NULL; - old_req = 1; + old_req = true; ibp->rvp.n_rc_dupreq++; spin_lock_irqsave(&qp->s_lock, flags); diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h index e00c8a7d559c..b5fc5c6cd52f 100644 --- a/drivers/infiniband/hw/hfi1/trace_ctxts.h +++ b/drivers/infiniband/hw/hfi1/trace_ctxts.h @@ -80,7 +80,7 @@ TRACE_EVENT(hfi1_uctxtdata, __entry->credits = uctxt->sc->credits; __entry->hw_free = le64_to_cpu(*uctxt->sc->hw_free); __entry->piobase = uctxt->sc->base_addr; - __entry->rcvhdrq_cnt = uctxt->rcvhdrq_cnt; + __entry->rcvhdrq_cnt = get_hdrq_cnt(uctxt); __entry->rcvhdrq_dma = uctxt->rcvhdrq_dma; __entry->eager_cnt = uctxt->egrbufs.alloced; __entry->rcvegr_dma = uctxt->egrbufs.rcvtids[0].dma; diff --git a/drivers/infiniband/hw/hfi1/trace_rx.h b/drivers/infiniband/hw/hfi1/trace_rx.h index 3cec960e9674..168079ed122c 100644 --- a/drivers/infiniband/hw/hfi1/trace_rx.h +++ b/drivers/infiniband/hw/hfi1/trace_rx.h @@ -106,19 +106,8 @@ TRACE_EVENT(hfi1_receive_interrupt, ), TP_fast_assign(DD_DEV_ASSIGN(dd); __entry->ctxt = rcd->ctxt; - if (rcd->do_interrupt == - &handle_receive_interrupt) { - __entry->slow_path = 1; - __entry->dma_rtail = 0xFF; - } else if (rcd->do_interrupt == - &handle_receive_interrupt_dma_rtail){ - __entry->dma_rtail = 1; - __entry->slow_path = 0; - } else if (rcd->do_interrupt == - &handle_receive_interrupt_nodma_rtail) { - __entry->dma_rtail = 0; - __entry->slow_path = 0; - } + __entry->slow_path = hfi1_is_slowpath(rcd); + __entry->dma_rtail = get_dma_rtail_setting(rcd); ), TP_printk("[%s] ctxt %d SlowPath: %d DmaRtail: %d", __get_str(dev), diff --git a/drivers/infiniband/hw/hfi1/vnic_main.c b/drivers/infiniband/hw/hfi1/vnic_main.c index b49e60e8397d..6b14581b9965 100644 --- a/drivers/infiniband/hw/hfi1/vnic_main.c +++ b/drivers/infiniband/hw/hfi1/vnic_main.c @@ -78,7 +78,7 @@ static int setup_vnic_ctxt(struct hfi1_devdata *dd, struct hfi1_ctxtdata *uctxt) if (ret) goto done; - if (uctxt->rcvhdrtail_kvaddr) + if (hfi1_rcvhdrtail_kvaddr(uctxt)) clear_rcvhdrtail(uctxt); rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB; diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index a2d1e5331bf1..5ffe4c996ed3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -370,6 +370,8 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, hr_cq->buf.size = hr_cq->cq_depth * hr_dev->caps.cq_entry_sz; hr_cq->buf.page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz; spin_lock_init(&hr_cq->lock); + INIT_LIST_HEAD(&hr_cq->sq_list); + INIT_LIST_HEAD(&hr_cq->rq_list); if (udata) { ret = create_user_cq(hr_dev, hr_cq, udata, &resp); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 5617434cbfb4..a7c4ff975c28 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -45,8 +45,6 @@ #define HNS_ROCE_MAX_MSG_LEN 0x80000000 -#define HNS_ROCE_ALIGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b)) - #define HNS_ROCE_IB_MIN_SQ_STRIDE 6 #define HNS_ROCE_BA_SIZE (32 * 4096) @@ -107,11 +105,6 @@ #define NODE_DESC_SIZE 64 #define DB_REG_OFFSET 0x1000 -#define SERV_TYPE_RC 0 -#define SERV_TYPE_RD 1 -#define SERV_TYPE_UC 2 -#define SERV_TYPE_UD 3 - /* Configure to HW for PAGE_SIZE larger than 4KB */ #define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) @@ -131,6 +124,13 @@ #define EQ_DEPTH_COEFF 2 enum { + SERV_TYPE_RC, + SERV_TYPE_UC, + SERV_TYPE_RD, + SERV_TYPE_UD, +}; + +enum { HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0, HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 << 1, }; @@ -423,7 +423,7 @@ struct hns_roce_mr_table { struct hns_roce_wq { u64 *wrid; /* Work request ID */ spinlock_t lock; - int wqe_cnt; /* WQE num */ + u32 wqe_cnt; /* WQE num */ int max_gs; int offset; int wqe_shift; /* WQE size */ @@ -498,6 +498,10 @@ struct hns_roce_cq { u32 vector; atomic_t refcount; struct completion free; + struct list_head sq_list; /* all qps on this send cq */ + struct list_head rq_list; /* all qps on this recv cq */ + int is_armed; /* cq is armed */ + struct list_head node; /* all armed cqs are on a list */ }; struct hns_roce_idx_que { @@ -647,7 +651,6 @@ struct hns_roce_qp { u8 sdb_en; u32 doorbell_qpn; u32 sq_signal_bits; - u32 sq_next_wqe; struct hns_roce_wq sq; struct ib_umem *umem; @@ -682,6 +685,9 @@ struct hns_roce_qp { u32 next_sge; struct hns_roce_rinl_buf rq_inl_buf; + struct list_head node; /* all qps are on a list */ + struct list_head rq_node; /* all recv qps are on a list */ + struct list_head sq_node; /* all send qps are on a list */ }; struct hns_roce_ib_iboe { @@ -794,10 +800,8 @@ struct hns_roce_caps { int reserved_qps; int num_qpc_timer; int num_cqc_timer; - u32 max_srq_sg; int num_srqs; u32 max_wqes; - u32 max_srqs; u32 max_srq_wrs; u32 max_srq_sges; u32 max_sq_desc_sz; @@ -811,7 +815,6 @@ struct hns_roce_caps { u32 min_wqes; int reserved_cqs; int reserved_srqs; - u32 max_srqwqes; int num_aeq_vectors; int num_comp_vectors; int num_other_vectors; @@ -895,6 +898,12 @@ struct hns_roce_caps { u32 tpq_buf_pg_sz; u32 chunk_sz; /* chunk size in non multihop mode */ u64 flags; + u16 default_ceq_max_cnt; + u16 default_ceq_period; + u16 default_aeq_max_cnt; + u16 default_aeq_period; + u16 default_aeq_arm_st; + u16 default_ceq_arm_st; }; struct hns_roce_work { @@ -911,6 +920,12 @@ struct hns_roce_dfx_hw { int *buffer); }; +enum hns_roce_device_state { + HNS_ROCE_DEVICE_STATE_INITED, + HNS_ROCE_DEVICE_STATE_RST_DOWN, + HNS_ROCE_DEVICE_STATE_UNINIT, +}; + struct hns_roce_hw { int (*reset)(struct hns_roce_dev *hr_dev, bool enable); int (*cmq_init)(struct hns_roce_dev *hr_dev); @@ -993,6 +1008,9 @@ struct hns_roce_dev { bool dis_db; unsigned long reset_cnt; struct hns_roce_ib_iboe iboe; + enum hns_roce_device_state state; + struct list_head qp_list; /* list of all qps on this dev */ + spinlock_t qp_list_lock; /* protect qp_list */ struct list_head pgdir_list; struct mutex pgdir_mutex; @@ -1133,7 +1151,6 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev); int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); -int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev); int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev); int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev); @@ -1257,6 +1274,7 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type); void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type); void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type); int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index); +void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev); int hns_roce_init(struct hns_roce_dev *hr_dev); void hns_roce_exit(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 2a2b2112f886..c6e66586e533 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -74,8 +74,8 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, unsigned long flags = 0; void *wqe = NULL; __le32 doorbell[2]; + u32 wqe_idx = 0; int nreq = 0; - u32 ind = 0; int ret = 0; u8 *smac; int loopback; @@ -88,7 +88,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, } spin_lock_irqsave(&qp->sq.lock, flags); - ind = qp->sq_next_wqe; + for (nreq = 0; wr; ++nreq, wr = wr->next) { if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { ret = -ENOMEM; @@ -96,6 +96,8 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, goto out; } + wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); + if (unlikely(wr->num_sge > qp->sq.max_gs)) { dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n", wr->num_sge, qp->sq.max_gs); @@ -104,9 +106,8 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, goto out; } - wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); - qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = - wr->wr_id; + wqe = get_send_wqe(qp, wqe_idx); + qp->sq.wrid[wqe_idx] = wr->wr_id; /* Corresponding to the RC and RD type wqe process separately */ if (ibqp->qp_type == IB_QPT_GSI) { @@ -210,7 +211,6 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, cpu_to_le32((wr->sg_list[1].addr) >> 32); ud_sq_wqe->l_key1 = cpu_to_le32(wr->sg_list[1].lkey); - ind++; } else if (ibqp->qp_type == IB_QPT_RC) { u32 tmp_len = 0; @@ -308,7 +308,6 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, ctrl->flag |= cpu_to_le32(wr->num_sge << HNS_ROCE_WQE_SGE_NUM_BIT); } - ind++; } } @@ -336,7 +335,6 @@ out: doorbell[1] = sq_db.u32_8; hns_roce_write64_k(doorbell, qp->sq.db_reg_l); - qp->sq_next_wqe = ind; } spin_unlock_irqrestore(&qp->sq.lock, flags); @@ -348,12 +346,6 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { - int ret = 0; - int nreq = 0; - int ind = 0; - int i = 0; - u32 reg_val; - unsigned long flags = 0; struct hns_roce_rq_wqe_ctrl *ctrl = NULL; struct hns_roce_wqe_data_seg *scat = NULL; struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); @@ -361,9 +353,14 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct device *dev = &hr_dev->pdev->dev; struct hns_roce_rq_db rq_db; __le32 doorbell[2] = {0}; + unsigned long flags = 0; + unsigned int wqe_idx; + int ret = 0; + int nreq = 0; + int i = 0; + u32 reg_val; spin_lock_irqsave(&hr_qp->rq.lock, flags); - ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1); for (nreq = 0; wr; ++nreq, wr = wr->next) { if (hns_roce_wq_overflow(&hr_qp->rq, nreq, @@ -373,6 +370,8 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp, goto out; } + wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); + if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n", wr->num_sge, hr_qp->rq.max_gs); @@ -381,7 +380,7 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp, goto out; } - ctrl = get_recv_wqe(hr_qp, ind); + ctrl = get_recv_wqe(hr_qp, wqe_idx); roce_set_field(ctrl->rwqe_byte_12, RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M, @@ -393,9 +392,7 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp, for (i = 0; i < wr->num_sge; i++) set_data_seg(scat + i, wr->sg_list + i); - hr_qp->rq.wrid[ind] = wr->wr_id; - - ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1); + hr_qp->rq.wrid[wqe_idx] = wr->wr_id; } out: @@ -2701,7 +2698,6 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, hr_qp->rq.tail = 0; hr_qp->sq.head = 0; hr_qp->sq.tail = 0; - hr_qp->sq_next_wqe = 0; } kfree(context); @@ -3315,7 +3311,6 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, hr_qp->rq.tail = 0; hr_qp->sq.head = 0; hr_qp->sq.tail = 0; - hr_qp->sq_next_wqe = 0; } out: kfree(context); @@ -3614,14 +3609,18 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (ret) return ret; - send_cq = to_hr_cq(hr_qp->ibqp.send_cq); - recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq); + send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; + recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; hns_roce_lock_cqs(send_cq, recv_cq); if (!udata) { - __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ? - to_hr_srq(hr_qp->ibqp.srq) : NULL); - if (send_cq != recv_cq) + if (recv_cq) + __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, + (hr_qp->ibqp.srq ? + to_hr_srq(hr_qp->ibqp.srq) : + NULL)); + + if (send_cq && send_cq != recv_cq) __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL); } hns_roce_unlock_cqs(send_cq, recv_cq); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index cb8071a3e0d5..12c4cd8e9378 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -63,20 +63,15 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, struct hns_roce_mr *mr = to_hr_mr(wr->mr); /* use ib_access_flags */ - roce_set_bit(rc_sq_wqe->byte_4, - V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S, + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S, wr->access & IB_ACCESS_MW_BIND ? 1 : 0); - roce_set_bit(rc_sq_wqe->byte_4, - V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S, + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S, wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); - roce_set_bit(rc_sq_wqe->byte_4, - V2_RC_FRMR_WQE_BYTE_4_RR_S, + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RR_S, wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0); - roce_set_bit(rc_sq_wqe->byte_4, - V2_RC_FRMR_WQE_BYTE_4_RW_S, + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RW_S, wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0); - roce_set_bit(rc_sq_wqe->byte_4, - V2_RC_FRMR_WQE_BYTE_4_LW_S, + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_LW_S, wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0); /* Data structure reuse may lead to confusion */ @@ -110,7 +105,7 @@ static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg, } static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, - unsigned int *sge_ind) + unsigned int *sge_ind, int valid_num_sge) { struct hns_roce_v2_wqe_data_seg *dseg; struct ib_sge *sg; @@ -123,7 +118,7 @@ static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; - extend_sge_num = wr->num_sge - num_in_wqe; + extend_sge_num = valid_num_sge - num_in_wqe; sg = wr->sg_list + num_in_wqe; shift = qp->hr_buf.page_shift; @@ -159,14 +154,16 @@ static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr, static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, void *wqe, unsigned int *sge_ind, + int valid_num_sge, const struct ib_send_wr **bad_wr) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_v2_wqe_data_seg *dseg = wqe; struct hns_roce_qp *qp = to_hr_qp(ibqp); + int j = 0; int i; - if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { + if (wr->send_flags & IB_SEND_INLINE && valid_num_sge) { if (le32_to_cpu(rc_sq_wqe->msg_len) > hr_dev->caps.max_sq_inline) { *bad_wr = wr; @@ -190,7 +187,7 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1); } else { - if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) { + if (valid_num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) { for (i = 0; i < wr->num_sge; i++) { if (likely(wr->sg_list[i].length)) { set_data_seg_v2(dseg, wr->sg_list + i); @@ -203,19 +200,21 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, (*sge_ind) & (qp->sge.sge_cnt - 1)); - for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) { + for (i = 0; i < wr->num_sge && + j < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) { if (likely(wr->sg_list[i].length)) { set_data_seg_v2(dseg, wr->sg_list + i); dseg++; + j++; } } - set_extend_sge(qp, wr, sge_ind); + set_extend_sge(qp, wr, sge_ind, valid_num_sge); } roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, - V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge); + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); } return 0; @@ -226,6 +225,30 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, int attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state); +static int check_send_valid(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct ib_qp *ibqp = &hr_qp->ibqp; + struct device *dev = hr_dev->dev; + + if (unlikely(ibqp->qp_type != IB_QPT_RC && + ibqp->qp_type != IB_QPT_GSI && + ibqp->qp_type != IB_QPT_UD)) { + dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type); + return -EOPNOTSUPP; + } else if (unlikely(hr_qp->state == IB_QPS_RESET || + hr_qp->state == IB_QPS_INIT || + hr_qp->state == IB_QPS_RTR)) { + dev_err(dev, "Post WQE fail, QP state %d!\n", hr_qp->state); + return -EINVAL; + } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) { + dev_err(dev, "Post WQE fail, dev state %d!\n", hr_dev->state); + return -EIO; + } + + return 0; +} + static int hns_roce_v2_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) @@ -239,38 +262,31 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct device *dev = hr_dev->dev; struct hns_roce_v2_db sq_db; struct ib_qp_attr attr; - unsigned int sge_ind; unsigned int owner_bit; + unsigned int sge_idx; + unsigned int wqe_idx; unsigned long flags; - unsigned int ind; + int valid_num_sge; void *wqe = NULL; bool loopback; int attr_mask; u32 tmp_len; - int ret = 0; u32 hr_op; u8 *smac; int nreq; + int ret; int i; - if (unlikely(ibqp->qp_type != IB_QPT_RC && - ibqp->qp_type != IB_QPT_GSI && - ibqp->qp_type != IB_QPT_UD)) { - dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type); - *bad_wr = wr; - return -EOPNOTSUPP; - } + spin_lock_irqsave(&qp->sq.lock, flags); - if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT || - qp->state == IB_QPS_RTR)) { - dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state); + ret = check_send_valid(hr_dev, qp); + if (ret) { *bad_wr = wr; - return -EINVAL; + nreq = 0; + goto out; } - spin_lock_irqsave(&qp->sq.lock, flags); - ind = qp->sq_next_wqe; - sge_ind = qp->next_sge; + sge_idx = qp->next_sge; for (nreq = 0; wr; ++nreq, wr = wr->next) { if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { @@ -279,6 +295,8 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, goto out; } + wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); + if (unlikely(wr->num_sge > qp->sq.max_gs)) { dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n", wr->num_sge, qp->sq.max_gs); @@ -287,14 +305,20 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, goto out; } - wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1)); - qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = - wr->wr_id; - + wqe = get_send_wqe(qp, wqe_idx); + qp->sq.wrid[wqe_idx] = wr->wr_id; owner_bit = ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); + valid_num_sge = 0; tmp_len = 0; + for (i = 0; i < wr->num_sge; i++) { + if (likely(wr->sg_list[i].length)) { + tmp_len += wr->sg_list[i].length; + valid_num_sge++; + } + } + /* Corresponding to the QP type, wqe process separately */ if (ibqp->qp_type == IB_QPT_GSI) { ud_sq_wqe = wqe; @@ -330,9 +354,6 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, V2_UD_SEND_WQE_BYTE_4_OPCODE_S, HNS_ROCE_V2_WQE_OP_SEND); - for (i = 0; i < wr->num_sge; i++) - tmp_len += wr->sg_list[i].length; - ud_sq_wqe->msg_len = cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len); @@ -368,12 +389,12 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, - wr->num_sge); + valid_num_sge); roce_set_field(ud_sq_wqe->byte_20, V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, - sge_ind & (qp->sge.sge_cnt - 1)); + sge_idx & (qp->sge.sge_cnt - 1)); roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M, @@ -423,13 +444,10 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2); - set_extend_sge(qp, wr, &sge_ind); - ind++; + set_extend_sge(qp, wr, &sge_idx, valid_num_sge); } else if (ibqp->qp_type == IB_QPT_RC) { rc_sq_wqe = wqe; memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe)); - for (i = 0; i < wr->num_sge; i++) - tmp_len += wr->sg_list[i].length; rc_sq_wqe->msg_len = cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len); @@ -550,15 +568,14 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, - wr->num_sge); + valid_num_sge); } else if (wr->opcode != IB_WR_REG_MR) { ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe, - wqe, &sge_ind, bad_wr); + wqe, &sge_idx, + valid_num_sge, bad_wr); if (ret) goto out; } - - ind++; } else { dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type); spin_unlock_irqrestore(&qp->sq.lock, flags); @@ -588,8 +605,7 @@ out: hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l); - qp->sq_next_wqe = ind; - qp->next_sge = sge_ind; + qp->next_sge = sge_idx; if (qp->state == IB_QPS_ERR) { attr_mask = IB_QP_STATE; @@ -610,6 +626,17 @@ out: return ret; } +static int check_recv_valid(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) + return -EIO; + else if (hr_qp->state == IB_QPS_RESET) + return -EINVAL; + + return 0; +} + static int hns_roce_v2_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) @@ -623,18 +650,18 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, unsigned long flags; void *wqe = NULL; int attr_mask; - int ret = 0; + u32 wqe_idx; int nreq; - int ind; + int ret; int i; spin_lock_irqsave(&hr_qp->rq.lock, flags); - ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1); - if (hr_qp->state == IB_QPS_RESET) { - spin_unlock_irqrestore(&hr_qp->rq.lock, flags); + ret = check_recv_valid(hr_dev, hr_qp); + if (ret) { *bad_wr = wr; - return -EINVAL; + nreq = 0; + goto out; } for (nreq = 0; wr; ++nreq, wr = wr->next) { @@ -645,6 +672,8 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, goto out; } + wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); + if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n", wr->num_sge, hr_qp->rq.max_gs); @@ -653,7 +682,7 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, goto out; } - wqe = get_recv_wqe(hr_qp, ind); + wqe = get_recv_wqe(hr_qp, wqe_idx); dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; for (i = 0; i < wr->num_sge; i++) { if (!wr->sg_list[i].length) @@ -669,8 +698,8 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, /* rq support inline data */ if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) { - sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list; - hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt = + sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list; + hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = (u32)wr->num_sge; for (i = 0; i < wr->num_sge; i++) { sge_list[i].addr = @@ -679,9 +708,7 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, } } - hr_qp->rq.wrid[ind] = wr->wr_id; - - ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1); + hr_qp->rq.wrid[wqe_idx] = wr->wr_id; } out: @@ -1254,7 +1281,6 @@ static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) } out: - dev_err(hr_dev->dev, "Func clear fail.\n"); hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag); } @@ -1378,8 +1404,7 @@ static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev) return 0; } -static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, - int vf_id) +static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id) { struct hns_roce_cmq_desc desc; struct hns_roce_vf_switch *swt; @@ -1388,13 +1413,12 @@ static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, swt = (struct hns_roce_vf_switch *)desc.data; hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true); swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL); - roce_set_field(swt->fun_id, - VF_SWITCH_DATA_FUN_ID_VF_ID_M, - VF_SWITCH_DATA_FUN_ID_VF_ID_S, - vf_id); + roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M, + VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id); ret = hns_roce_cmq_send(hr_dev, &desc, 1); if (ret) return ret; + desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); @@ -1574,69 +1598,9 @@ static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) return hns_roce_cmq_send(hr_dev, &desc, 1); } -static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) +static void set_default_caps(struct hns_roce_dev *hr_dev) { struct hns_roce_caps *caps = &hr_dev->caps; - int ret; - - ret = hns_roce_cmq_query_hw_info(hr_dev); - if (ret) { - dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n", - ret); - return ret; - } - - ret = hns_roce_query_fw_ver(hr_dev); - if (ret) { - dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n", - ret); - return ret; - } - - ret = hns_roce_config_global_param(hr_dev); - if (ret) { - dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", - ret); - return ret; - } - - /* Get pf resource owned by every pf */ - ret = hns_roce_query_pf_resource(hr_dev); - if (ret) { - dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n", - ret); - return ret; - } - - if (hr_dev->pci_dev->revision == 0x21) { - ret = hns_roce_query_pf_timer_resource(hr_dev); - if (ret) { - dev_err(hr_dev->dev, - "Query pf timer resource fail, ret = %d.\n", - ret); - return ret; - } - } - - ret = hns_roce_alloc_vf_resource(hr_dev); - if (ret) { - dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n", - ret); - return ret; - } - - if (hr_dev->pci_dev->revision == 0x21) { - ret = hns_roce_set_vf_switch_param(hr_dev, 0); - if (ret) { - dev_err(hr_dev->dev, - "Set function switch param fail, ret = %d.\n", - ret); - return ret; - } - } - - hr_dev->vendor_part_id = hr_dev->pci_dev->device; - hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM; @@ -1644,17 +1608,15 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM; caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; - caps->max_srqwqes = HNS_ROCE_V2_MAX_SRQWQE_NUM; caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM; caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; - caps->max_srq_sg = HNS_ROCE_V2_MAX_SRQ_SGE_NUM; caps->num_uars = HNS_ROCE_V2_UAR_NUM; caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM; - caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; + caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; @@ -1668,12 +1630,12 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ; caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; - caps->trrl_entry_sz = HNS_ROCE_V2_TRRL_ENTRY_SZ; + caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ; caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; - caps->idx_entry_sz = 4; + caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ; caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE; caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; caps->reserved_lkey = 0; @@ -1696,16 +1658,13 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->mpt_ba_pg_sz = 0; caps->mpt_buf_pg_sz = 0; caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->pbl_ba_pg_sz = 2; - caps->pbl_buf_pg_sz = 0; - caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; caps->mtt_ba_pg_sz = 0; caps->mtt_buf_pg_sz = 0; caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; - caps->wqe_sq_hop_num = 2; - caps->wqe_sge_hop_num = 1; - caps->wqe_rq_hop_num = 2; - caps->cqe_ba_pg_sz = 6; + caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM; + caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM; + caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM; + caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K; caps->cqe_buf_pg_sz = 0; caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; caps->srqwqe_ba_pg_sz = 0; @@ -1714,10 +1673,6 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->idx_ba_pg_sz = 0; caps->idx_buf_pg_sz = 0; caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM; - caps->eqe_ba_pg_sz = 0; - caps->eqe_buf_pg_sz = 0; - caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; - caps->tsq_buf_pg_sz = 0; caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | @@ -1726,24 +1681,19 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) HNS_ROCE_CAP_FLAG_RECORD_DB | HNS_ROCE_CAP_FLAG_SQ_RECORD_DB; - if (hr_dev->pci_dev->revision == 0x21) - caps->flags |= HNS_ROCE_CAP_FLAG_MW | - HNS_ROCE_CAP_FLAG_FRMR; - caps->pkey_table_len[0] = 1; - caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; + caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; caps->local_ca_ack_delay = 0; caps->max_mtu = IB_MTU_4096; - caps->max_srqs = HNS_ROCE_V2_MAX_SRQ; caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR; caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE; - if (hr_dev->pci_dev->revision == 0x21) { - caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | - HNS_ROCE_CAP_FLAG_SRQ | + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B) { + caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | + HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; @@ -1757,12 +1707,345 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->cqc_timer_buf_pg_sz = 0; caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; - caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ; - caps->sccc_ba_pg_sz = 0; - caps->sccc_buf_pg_sz = 0; - caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; + caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ; + caps->sccc_ba_pg_sz = 0; + caps->sccc_buf_pg_sz = 0; + caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; + } +} + +static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num, + int *buf_page_size, int *bt_page_size, u32 hem_type) +{ + u64 obj_per_chunk; + int bt_chunk_size = 1 << PAGE_SHIFT; + int buf_chunk_size = 1 << PAGE_SHIFT; + int obj_per_chunk_default = buf_chunk_size / obj_size; + + *buf_page_size = 0; + *bt_page_size = 0; + + switch (hop_num) { + case 3: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 2: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 1: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case HNS_ROCE_HOP_NUM_0: + obj_per_chunk = ctx_bt_num * obj_per_chunk_default; + break; + default: + pr_err("Table %d not support hop_num = %d!\n", hem_type, + hop_num); + return; + } + + if (hem_type >= HEM_TYPE_MTT) + *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); + else + *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); +} + +static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM]; + struct hns_roce_caps *caps = &hr_dev->caps; + struct hns_roce_query_pf_caps_a *resp_a; + struct hns_roce_query_pf_caps_b *resp_b; + struct hns_roce_query_pf_caps_c *resp_c; + struct hns_roce_query_pf_caps_d *resp_d; + struct hns_roce_query_pf_caps_e *resp_e; + int ctx_hop_num; + int pbl_hop_num; + int ret; + int i; + + for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) { + hns_roce_cmq_setup_basic_desc(&desc[i], + HNS_ROCE_OPC_QUERY_PF_CAPS_NUM, + true); + if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1)) + desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + } + + ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM); + if (ret) + return ret; + + resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data; + resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data; + resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data; + resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data; + resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data; + + caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; + caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); + caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); + caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); + caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg); + caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer); + caps->num_cqc_timer = le16_to_cpu(resp_a->num_cqc_timer); + caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); + caps->num_aeq_vectors = resp_a->num_aeq_vectors; + caps->num_other_vectors = resp_a->num_other_vectors; + caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; + caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; + caps->max_srq_desc_sz = resp_a->max_srq_desc_sz; + caps->cq_entry_sz = resp_a->cq_entry_sz; + + caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; + caps->irrl_entry_sz = resp_b->irrl_entry_sz; + caps->trrl_entry_sz = resp_b->trrl_entry_sz; + caps->cqc_entry_sz = resp_b->cqc_entry_sz; + caps->srqc_entry_sz = resp_b->srqc_entry_sz; + caps->idx_entry_sz = resp_b->idx_entry_sz; + caps->sccc_entry_sz = resp_b->scc_ctx_entry_sz; + caps->max_mtu = resp_b->max_mtu; + caps->qpc_entry_sz = le16_to_cpu(resp_b->qpc_entry_sz); + caps->min_cqes = resp_b->min_cqes; + caps->min_wqes = resp_b->min_wqes; + caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); + caps->pkey_table_len[0] = resp_b->pkey_table_len; + caps->phy_num_uars = resp_b->phy_num_uars; + ctx_hop_num = resp_b->ctx_hop_num; + pbl_hop_num = resp_b->pbl_hop_num; + + caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds, + V2_QUERY_PF_CAPS_C_NUM_PDS_M, + V2_QUERY_PF_CAPS_C_NUM_PDS_S); + caps->flags = roce_get_field(resp_c->cap_flags_num_pds, + V2_QUERY_PF_CAPS_C_CAP_FLAGS_M, + V2_QUERY_PF_CAPS_C_CAP_FLAGS_S); + caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs, + V2_QUERY_PF_CAPS_C_NUM_CQS_M, + V2_QUERY_PF_CAPS_C_NUM_CQS_S); + caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs, + V2_QUERY_PF_CAPS_C_MAX_GID_M, + V2_QUERY_PF_CAPS_C_MAX_GID_S); + caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth, + V2_QUERY_PF_CAPS_C_CQ_DEPTH_M, + V2_QUERY_PF_CAPS_C_CQ_DEPTH_S); + caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws, + V2_QUERY_PF_CAPS_C_NUM_MRWS_M, + V2_QUERY_PF_CAPS_C_NUM_MRWS_S); + caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps, + V2_QUERY_PF_CAPS_C_NUM_QPS_M, + V2_QUERY_PF_CAPS_C_NUM_QPS_S); + caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps, + V2_QUERY_PF_CAPS_C_MAX_ORD_M, + V2_QUERY_PF_CAPS_C_MAX_ORD_S); + caps->max_qp_dest_rdma = caps->max_qp_init_rdma; + caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth); + caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_NUM_SRQS_M, + V2_QUERY_PF_CAPS_D_NUM_SRQS_S); + caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); + caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth, + V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M, + V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S); + caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth, + V2_QUERY_PF_CAPS_D_NUM_CEQS_M, + V2_QUERY_PF_CAPS_D_NUM_CEQS_S); + caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth, + V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M, + V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S); + caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth, + V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M, + V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S); + caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth, + V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M, + V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S); + caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds, + V2_QUERY_PF_CAPS_D_RSV_PDS_M, + V2_QUERY_PF_CAPS_D_RSV_PDS_S); + caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds, + V2_QUERY_PF_CAPS_D_NUM_UARS_M, + V2_QUERY_PF_CAPS_D_NUM_UARS_S); + caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps, + V2_QUERY_PF_CAPS_D_RSV_QPS_M, + V2_QUERY_PF_CAPS_D_RSV_QPS_S); + caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps, + V2_QUERY_PF_CAPS_D_RSV_UARS_M, + V2_QUERY_PF_CAPS_D_RSV_UARS_S); + caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws, + V2_QUERY_PF_CAPS_E_RSV_MRWS_M, + V2_QUERY_PF_CAPS_E_RSV_MRWS_S); + caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws, + V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M, + V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S); + caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs, + V2_QUERY_PF_CAPS_E_RSV_CQS_M, + V2_QUERY_PF_CAPS_E_RSV_CQS_S); + caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs, + V2_QUERY_PF_CAPS_E_RSV_SRQS_M, + V2_QUERY_PF_CAPS_E_RSV_SRQS_S); + caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey, + V2_QUERY_PF_CAPS_E_RSV_LKEYS_M, + V2_QUERY_PF_CAPS_E_RSV_LKEYS_S); + caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); + caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); + caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); + caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); + + caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; + caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; + caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; + caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; + caps->mtt_ba_pg_sz = 0; + caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; + caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; + caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; + + caps->qpc_hop_num = ctx_hop_num; + caps->srqc_hop_num = ctx_hop_num; + caps->cqc_hop_num = ctx_hop_num; + caps->mpt_hop_num = ctx_hop_num; + caps->mtt_hop_num = pbl_hop_num; + caps->cqe_hop_num = pbl_hop_num; + caps->srqwqe_hop_num = pbl_hop_num; + caps->idx_hop_num = pbl_hop_num; + caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M, + V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S); + caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M, + V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S); + caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M, + V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); + + calc_pg_sz(caps->num_qps, caps->qpc_entry_sz, caps->qpc_hop_num, + caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, + HEM_TYPE_QPC); + calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, + caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, + HEM_TYPE_MTPT); + calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, + caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, + HEM_TYPE_CQC); + calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num, + caps->srqc_bt_num, &caps->srqc_buf_pg_sz, + &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC); + + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B) { + caps->sccc_hop_num = ctx_hop_num; + caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; + caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; + + calc_pg_sz(caps->num_qps, caps->sccc_entry_sz, + caps->sccc_hop_num, caps->sccc_bt_num, + &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz, + HEM_TYPE_SCCC); + calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz, + caps->cqc_timer_hop_num, caps->cqc_timer_bt_num, + &caps->cqc_timer_buf_pg_sz, + &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER); + } + + calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num, + 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); + calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, + caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, + &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); + calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num, + 1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX); + + return 0; +} + +static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_caps *caps = &hr_dev->caps; + int ret; + + ret = hns_roce_cmq_query_hw_info(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_query_fw_ver(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_config_global_param(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", + ret); + return ret; + } + + /* Get pf resource owned by every pf */ + ret = hns_roce_query_pf_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n", + ret); + return ret; + } + + if (hr_dev->pci_dev->revision == 0x21) { + ret = hns_roce_query_pf_timer_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, + "Query pf timer resource fail, ret = %d.\n", + ret); + return ret; + } + } + + ret = hns_roce_alloc_vf_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n", + ret); + return ret; + } + + if (hr_dev->pci_dev->revision == 0x21) { + ret = hns_roce_set_vf_switch_param(hr_dev, 0); + if (ret) { + dev_err(hr_dev->dev, + "Set function switch param fail, ret = %d.\n", + ret); + return ret; + } } + hr_dev->vendor_part_id = hr_dev->pci_dev->device; + hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); + + caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; + caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; + caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; + caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; + + caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K; + caps->pbl_buf_pg_sz = 0; + caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; + caps->eqe_ba_pg_sz = 0; + caps->eqe_buf_pg_sz = 0; + caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; + caps->tsq_buf_pg_sz = 0; + + ret = hns_roce_query_pf_caps(hr_dev); + if (ret) + set_default_caps(hr_dev); + ret = hns_roce_v2_set_bt(hr_dev); if (ret) dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n", @@ -1818,37 +2101,32 @@ static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev, req_a->base_addr_h = cpu_to_le32(link_tbl->table.map >> 32); roce_set_field(req_a->depth_pgsz_init_en, - CFG_LLM_QUE_DEPTH_M, - CFG_LLM_QUE_DEPTH_S, + CFG_LLM_QUE_DEPTH_M, CFG_LLM_QUE_DEPTH_S, link_tbl->npages); roce_set_field(req_a->depth_pgsz_init_en, - CFG_LLM_QUE_PGSZ_M, - CFG_LLM_QUE_PGSZ_S, + CFG_LLM_QUE_PGSZ_M, CFG_LLM_QUE_PGSZ_S, link_tbl->pg_sz); req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0); req_a->head_ba_h_nxtptr = cpu_to_le32(entry[0].blk_ba1_nxt_ptr); - roce_set_field(req_a->head_ptr, - CFG_LLM_HEAD_PTR_M, + roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, CFG_LLM_HEAD_PTR_S, 0); } else { req_b->tail_ba_l = cpu_to_le32(entry[page_num - 1].blk_ba0); - roce_set_field(req_b->tail_ba_h, - CFG_LLM_TAIL_BA_H_M, + roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M, CFG_LLM_TAIL_BA_H_S, entry[page_num - 1].blk_ba1_nxt_ptr & - HNS_ROCE_LINK_TABLE_BA1_M); - roce_set_field(req_b->tail_ptr, - CFG_LLM_TAIL_PTR_M, + HNS_ROCE_LINK_TABLE_BA1_M); + roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, CFG_LLM_TAIL_PTR_S, (entry[page_num - 2].blk_ba1_nxt_ptr & - HNS_ROCE_LINK_TABLE_NXT_PTR_M) >> - HNS_ROCE_LINK_TABLE_NXT_PTR_S); + HNS_ROCE_LINK_TABLE_NXT_PTR_M) >> + HNS_ROCE_LINK_TABLE_NXT_PTR_S); } } - roce_set_field(req_a->depth_pgsz_init_en, - CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1); + roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M, + CFG_LLM_INIT_EN_S, 1); return hns_roce_cmq_send(hr_dev, desc, 2); } @@ -2141,11 +2419,9 @@ static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev, hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false); - roce_set_field(sgid_tb->table_idx_rsv, - CFG_SGID_TB_TABLE_IDX_M, + roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M, CFG_SGID_TB_TABLE_IDX_S, gid_index); - roce_set_field(sgid_tb->vf_sgid_type_rsv, - CFG_SGID_TB_VF_SGID_TYPE_M, + roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M, CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type); p = (u32 *)&gid->raw[0]; @@ -2416,11 +2692,10 @@ static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE); roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, V2_MPT_BYTE_4_PD_S, mw->pdn); - roce_set_field(mpt_entry->byte_4_pd_hop_st, - V2_MPT_BYTE_4_PBL_HOP_NUM_M, + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, V2_MPT_BYTE_4_PBL_HOP_NUM_S, - mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? - 0 : mw->pbl_hop_num); + mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : + mw->pbl_hop_num); roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, @@ -2561,8 +2836,7 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M, V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE); roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M, - V2_CQC_BYTE_4_SHIFT_S, - ilog2(hr_cq->cq_depth)); + V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth)); roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M, V2_CQC_BYTE_4_CEQN_S, hr_cq->vector); @@ -2687,6 +2961,55 @@ static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, return 0; } +static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq, + int num_entries, struct ib_wc *wc) +{ + unsigned int left; + int npolled = 0; + + left = wq->head - wq->tail; + if (left == 0) + return 0; + + left = min_t(unsigned int, (unsigned int)num_entries, left); + while (npolled < left) { + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + wc->status = IB_WC_WR_FLUSH_ERR; + wc->vendor_err = 0; + wc->qp = &hr_qp->ibqp; + + wq->tail++; + wc++; + npolled++; + } + + return npolled; +} + +static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries, + struct ib_wc *wc) +{ + struct hns_roce_qp *hr_qp; + int npolled = 0; + + list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { + npolled += sw_comp(hr_qp, &hr_qp->sq, + num_entries - npolled, wc + npolled); + if (npolled >= num_entries) + goto out; + } + + list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { + npolled += sw_comp(hr_qp, &hr_qp->rq, + num_entries - npolled, wc + npolled); + if (npolled >= num_entries) + goto out; + } + +out: + return npolled; +} + static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, struct hns_roce_qp **cur_qp, struct ib_wc *wc) { @@ -2967,6 +3290,7 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { + struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); struct hns_roce_qp *cur_qp = NULL; unsigned long flags; @@ -2974,6 +3298,18 @@ static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, spin_lock_irqsave(&hr_cq->lock, flags); + /* + * When the device starts to reset, the state is RST_DOWN. At this time, + * there may still be some valid CQEs in the hardware that are not + * polled. Therefore, it is not allowed to switch to the software mode + * immediately. When the state changes to UNINIT, CQE no longer exists + * in the hardware, and then switch to software mode. + */ + if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) { + npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc); + goto out; + } + for (npolled = 0; npolled < num_entries; ++npolled) { if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) break; @@ -2985,6 +3321,7 @@ static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); } +out: spin_unlock_irqrestore(&hr_cq->lock, flags); return npolled; @@ -3159,8 +3496,6 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, } static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, - enum ib_qp_state cur_state, - enum ib_qp_state new_state, struct hns_roce_v2_qp_context *context, struct hns_roce_qp *hr_qp) { @@ -3210,6 +3545,9 @@ static void set_access_flags(struct hns_roce_qp *hr_qp, roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0); + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, + !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 0); } static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, @@ -3577,6 +3915,12 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, IB_ACCESS_REMOTE_ATOMIC)); roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0); + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, 0); } else { roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ)); @@ -3592,6 +3936,11 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0); + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, 0); } roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, @@ -3838,13 +4187,11 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, /* Configure GID index */ port_num = rdma_ah_get_port_num(&attr->ah_attr); roce_set_field(context->byte_20_smac_sgid_idx, - V2_QPC_BYTE_20_SGID_IDX_M, - V2_QPC_BYTE_20_SGID_IDX_S, + V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, hns_get_gid_index(hr_dev, port_num - 1, grh->sgid_index)); roce_set_field(qpc_mask->byte_20_smac_sgid_idx, - V2_QPC_BYTE_20_SGID_IDX_M, - V2_QPC_BYTE_20_SGID_IDX_S, 0); + V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0); memcpy(&(context->dmac), dmac, sizeof(u32)); roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4]))); @@ -4234,8 +4581,7 @@ static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp, roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M, - V2_QPC_BYTE_212_RETRY_CNT_S, - attr->retry_cnt); + V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt); roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M, V2_QPC_BYTE_212_RETRY_CNT_S, 0); @@ -4443,7 +4789,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, V2_QPC_BYTE_60_QP_ST_S, 0); /* SW pass context to HW */ - ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state, ctx, hr_qp); + ret = hns_roce_v2_qp_modify(hr_dev, ctx, hr_qp); if (ret) { dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret); goto out; @@ -4464,7 +4810,6 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, hr_qp->rq.tail = 0; hr_qp->sq.head = 0; hr_qp->sq.tail = 0; - hr_qp->sq_next_wqe = 0; hr_qp->next_sge = 0; if (hr_qp->rq.wqe_cnt) *hr_qp->rdb.db_record = 0; @@ -4649,6 +4994,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, { struct hns_roce_cq *send_cq, *recv_cq; struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned long flags; int ret = 0; if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) { @@ -4659,21 +5005,32 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, ibdev_err(ibdev, "modify QP to Reset failed.\n"); } - send_cq = to_hr_cq(hr_qp->ibqp.send_cq); - recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq); + send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; + recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; + spin_lock_irqsave(&hr_dev->qp_list_lock, flags); hns_roce_lock_cqs(send_cq, recv_cq); + list_del(&hr_qp->node); + list_del(&hr_qp->sq_node); + list_del(&hr_qp->rq_node); + if (!udata) { - __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ? - to_hr_srq(hr_qp->ibqp.srq) : NULL); - if (send_cq != recv_cq) + if (recv_cq) + __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, + (hr_qp->ibqp.srq ? + to_hr_srq(hr_qp->ibqp.srq) : + NULL)); + + if (send_cq && send_cq != recv_cq) __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL); + } hns_roce_qp_remove(hr_dev, hr_qp); hns_roce_unlock_cqs(send_cq, recv_cq); + spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); hns_roce_qp_free(hr_dev, hr_qp); @@ -5155,8 +5512,7 @@ static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, */ dma_rmb(); - cqn = roce_get_field(ceqe->comp, - HNS_ROCE_V2_CEQE_COMP_CQN_M, + cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M, HNS_ROCE_V2_CEQE_COMP_CQN_S); hns_roce_cq_completion(hr_dev, cqn); @@ -5409,126 +5765,98 @@ static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev, eq->eqe_ba = eq->l0_dma; /* set eqc state */ - roce_set_field(eqc->byte_4, - HNS_ROCE_EQC_EQ_ST_M, - HNS_ROCE_EQC_EQ_ST_S, + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S, HNS_ROCE_V2_EQ_STATE_VALID); /* set eqe hop num */ - roce_set_field(eqc->byte_4, - HNS_ROCE_EQC_HOP_NUM_M, + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M, HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num); /* set eqc over_ignore */ - roce_set_field(eqc->byte_4, - HNS_ROCE_EQC_OVER_IGNORE_M, + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M, HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore); /* set eqc coalesce */ - roce_set_field(eqc->byte_4, - HNS_ROCE_EQC_COALESCE_M, + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M, HNS_ROCE_EQC_COALESCE_S, eq->coalesce); /* set eqc arm_state */ - roce_set_field(eqc->byte_4, - HNS_ROCE_EQC_ARM_ST_M, + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M, HNS_ROCE_EQC_ARM_ST_S, eq->arm_st); /* set eqn */ - roce_set_field(eqc->byte_4, - HNS_ROCE_EQC_EQN_M, - HNS_ROCE_EQC_EQN_S, eq->eqn); + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S, + eq->eqn); /* set eqe_cnt */ - roce_set_field(eqc->byte_4, - HNS_ROCE_EQC_EQE_CNT_M, - HNS_ROCE_EQC_EQE_CNT_S, - HNS_ROCE_EQ_INIT_EQE_CNT); + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M, + HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT); /* set eqe_ba_pg_sz */ - roce_set_field(eqc->byte_8, - HNS_ROCE_EQC_BA_PG_SZ_M, + roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M, HNS_ROCE_EQC_BA_PG_SZ_S, eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET); /* set eqe_buf_pg_sz */ - roce_set_field(eqc->byte_8, - HNS_ROCE_EQC_BUF_PG_SZ_M, + roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M, HNS_ROCE_EQC_BUF_PG_SZ_S, eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET); /* set eq_producer_idx */ - roce_set_field(eqc->byte_8, - HNS_ROCE_EQC_PROD_INDX_M, - HNS_ROCE_EQC_PROD_INDX_S, - HNS_ROCE_EQ_INIT_PROD_IDX); + roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M, + HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX); /* set eq_max_cnt */ - roce_set_field(eqc->byte_12, - HNS_ROCE_EQC_MAX_CNT_M, + roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M, HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt); /* set eq_period */ - roce_set_field(eqc->byte_12, - HNS_ROCE_EQC_PERIOD_M, + roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M, HNS_ROCE_EQC_PERIOD_S, eq->eq_period); /* set eqe_report_timer */ - roce_set_field(eqc->eqe_report_timer, - HNS_ROCE_EQC_REPORT_TIMER_M, + roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M, HNS_ROCE_EQC_REPORT_TIMER_S, HNS_ROCE_EQ_INIT_REPORT_TIMER); /* set eqe_ba [34:3] */ - roce_set_field(eqc->eqe_ba0, - HNS_ROCE_EQC_EQE_BA_L_M, + roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M, HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3); /* set eqe_ba [64:35] */ - roce_set_field(eqc->eqe_ba1, - HNS_ROCE_EQC_EQE_BA_H_M, + roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M, HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35); /* set eq shift */ - roce_set_field(eqc->byte_28, - HNS_ROCE_EQC_SHIFT_M, - HNS_ROCE_EQC_SHIFT_S, eq->shift); + roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S, + eq->shift); /* set eq MSI_IDX */ - roce_set_field(eqc->byte_28, - HNS_ROCE_EQC_MSI_INDX_M, - HNS_ROCE_EQC_MSI_INDX_S, - HNS_ROCE_EQ_INIT_MSI_IDX); + roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M, + HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX); /* set cur_eqe_ba [27:12] */ - roce_set_field(eqc->byte_28, - HNS_ROCE_EQC_CUR_EQE_BA_L_M, + roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M, HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12); /* set cur_eqe_ba [59:28] */ - roce_set_field(eqc->byte_32, - HNS_ROCE_EQC_CUR_EQE_BA_M_M, + roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M, HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28); /* set cur_eqe_ba [63:60] */ - roce_set_field(eqc->byte_36, - HNS_ROCE_EQC_CUR_EQE_BA_H_M, + roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M, HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60); /* set eq consumer idx */ - roce_set_field(eqc->byte_36, - HNS_ROCE_EQC_CONS_INDX_M, - HNS_ROCE_EQC_CONS_INDX_S, - HNS_ROCE_EQ_INIT_CONS_IDX); + roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M, + HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX); /* set nex_eqe_ba[43:12] */ - roce_set_field(eqc->nxt_eqe_ba0, - HNS_ROCE_EQC_NXT_EQE_BA_L_M, + roce_set_field(eqc->nxt_eqe_ba0, HNS_ROCE_EQC_NXT_EQE_BA_L_M, HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12); /* set nex_eqe_ba[63:44] */ - roce_set_field(eqc->nxt_eqe_ba1, - HNS_ROCE_EQC_NXT_EQE_BA_H_M, + roce_set_field(eqc->nxt_eqe_ba1, HNS_ROCE_EQC_NXT_EQE_BA_H_M, HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44); } @@ -5828,18 +6156,16 @@ static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num, /* irq contains: abnormal + AEQ + CEQ */ for (j = 0; j < other_num; j++) - snprintf((char *)hr_dev->irq_names[j], - HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", j); + snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, + "hns-abn-%d", j); for (j = other_num; j < (other_num + aeq_num); j++) - snprintf((char *)hr_dev->irq_names[j], - HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d", - j - other_num); + snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, + "hns-aeq-%d", j - other_num); for (j = (other_num + aeq_num); j < irq_num; j++) - snprintf((char *)hr_dev->irq_names[j], - HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d", - j - other_num - aeq_num); + snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, + "hns-ceq-%d", j - other_num - aeq_num); for (j = 0; j < irq_num; j++) { if (j < other_num) @@ -6448,6 +6774,10 @@ static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, return; handle->priv = NULL; + + hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT; + hns_roce_handle_device_err(hr_dev); + hns_roce_exit(hr_dev); kfree(hr_dev->priv); ib_dealloc_device(&hr_dev->ib_dev); @@ -6509,7 +6839,6 @@ static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) { struct hns_roce_dev *hr_dev; - struct ib_event event; if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) { set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); @@ -6527,10 +6856,7 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) hr_dev->active = false; hr_dev->dis_db = true; - event.event = IB_EVENT_DEVICE_FATAL; - event.device = &hr_dev->ib_dev; - event.element.port_num = 1; - ib_dispatch_event(&event); + hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN; return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 76a14db7028d..2a117ff6a6be 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -81,10 +81,12 @@ #define HNS_ROCE_V2_QPC_ENTRY_SZ 256 #define HNS_ROCE_V2_IRRL_ENTRY_SZ 64 #define HNS_ROCE_V2_TRRL_ENTRY_SZ 48 +#define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100 #define HNS_ROCE_V2_CQC_ENTRY_SZ 64 #define HNS_ROCE_V2_SRQC_ENTRY_SZ 64 #define HNS_ROCE_V2_MTPT_ENTRY_SZ 64 #define HNS_ROCE_V2_MTT_ENTRY_SZ 64 +#define HNS_ROCE_V2_IDX_ENTRY_SZ 4 #define HNS_ROCE_V2_CQE_ENTRY_SIZE 32 #define HNS_ROCE_V2_SCCC_ENTRY_SZ 32 #define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ PAGE_SIZE @@ -109,7 +111,12 @@ #define HNS_ROCE_PBL_HOP_NUM 2 #define HNS_ROCE_EQE_HOP_NUM 2 #define HNS_ROCE_IDX_HOP_NUM 1 +#define HNS_ROCE_SQWQE_HOP_NUM 2 +#define HNS_ROCE_EXT_SGE_HOP_NUM 1 +#define HNS_ROCE_RQWQE_HOP_NUM 2 +#define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K 6 +#define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K 2 #define HNS_ROCE_V2_GID_INDEX_NUM 256 #define HNS_ROCE_V2_TABLE_CHUNK_SIZE (1 << 18) @@ -237,6 +244,7 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_CFG_EXT_LLM = 0x8403, HNS_ROCE_OPC_CFG_TMOUT_LLM = 0x8404, HNS_ROCE_OPC_QUERY_PF_TIMER_RES = 0x8406, + HNS_ROCE_OPC_QUERY_PF_CAPS_NUM = 0x8408, HNS_ROCE_OPC_CFG_SGID_TB = 0x8500, HNS_ROCE_OPC_CFG_SMAC_TB = 0x8501, HNS_ROCE_OPC_POST_MB = 0x8504, @@ -643,7 +651,7 @@ struct hns_roce_v2_qp_context { #define V2_QPC_BYTE_76_ATE_S 27 #define V2_QPC_BYTE_76_RQIE_S 28 - +#define V2_QPC_BYTE_76_EXT_ATE_S 29 #define V2_QPC_BYTE_76_RQ_VLAN_EN_S 30 #define V2_QPC_BYTE_80_RX_CQN_S 0 #define V2_QPC_BYTE_80_RX_CQN_M GENMASK(23, 0) @@ -1569,6 +1577,155 @@ struct hns_roce_cfg_smac_tb { #define CFG_SMAC_TB_VF_SMAC_H_S 0 #define CFG_SMAC_TB_VF_SMAC_H_M GENMASK(15, 0) +#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM 5 +struct hns_roce_query_pf_caps_a { + u8 number_ports; + u8 local_ca_ack_delay; + __le16 max_sq_sg; + __le16 max_sq_inline; + __le16 max_rq_sg; + __le32 max_extend_sg; + __le16 num_qpc_timer; + __le16 num_cqc_timer; + __le16 max_srq_sges; + u8 num_aeq_vectors; + u8 num_other_vectors; + u8 max_sq_desc_sz; + u8 max_rq_desc_sz; + u8 max_srq_desc_sz; + u8 cq_entry_sz; +}; + +struct hns_roce_query_pf_caps_b { + u8 mtpt_entry_sz; + u8 irrl_entry_sz; + u8 trrl_entry_sz; + u8 cqc_entry_sz; + u8 srqc_entry_sz; + u8 idx_entry_sz; + u8 scc_ctx_entry_sz; + u8 max_mtu; + __le16 qpc_entry_sz; + __le16 qpc_timer_entry_sz; + __le16 cqc_timer_entry_sz; + u8 min_cqes; + u8 min_wqes; + __le32 page_size_cap; + u8 pkey_table_len; + u8 phy_num_uars; + u8 ctx_hop_num; + u8 pbl_hop_num; +}; + +struct hns_roce_query_pf_caps_c { + __le32 cap_flags_num_pds; + __le32 max_gid_num_cqs; + __le32 cq_depth; + __le32 num_mrws; + __le32 ord_num_qps; + __le16 sq_depth; + __le16 rq_depth; +}; + +#define V2_QUERY_PF_CAPS_C_NUM_PDS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_PDS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_CAP_FLAGS_S 20 +#define V2_QUERY_PF_CAPS_C_CAP_FLAGS_M GENMASK(31, 20) + +#define V2_QUERY_PF_CAPS_C_NUM_CQS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_CQS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_MAX_GID_S 20 +#define V2_QUERY_PF_CAPS_C_MAX_GID_M GENMASK(28, 20) + +#define V2_QUERY_PF_CAPS_C_CQ_DEPTH_S 0 +#define V2_QUERY_PF_CAPS_C_CQ_DEPTH_M GENMASK(22, 0) + +#define V2_QUERY_PF_CAPS_C_NUM_MRWS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_MRWS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_NUM_QPS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_QPS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_MAX_ORD_S 20 +#define V2_QUERY_PF_CAPS_C_MAX_ORD_M GENMASK(27, 20) + +struct hns_roce_query_pf_caps_d { + __le32 wq_hop_num_max_srqs; + __le16 srq_depth; + __le16 rsv; + __le32 num_ceqs_ceq_depth; + __le32 arm_st_aeq_depth; + __le32 num_uars_rsv_pds; + __le32 rsv_uars_rsv_qps; +}; +#define V2_QUERY_PF_CAPS_D_NUM_SRQS_S 0 +#define V2_QUERY_PF_CAPS_D_NUM_SRQS_M GENMASK(20, 0) + +#define V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S 20 +#define V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M GENMASK(21, 20) + +#define V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S 22 +#define V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M GENMASK(23, 22) + +#define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S 24 +#define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M GENMASK(25, 24) + + +#define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S 0 +#define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M GENMASK(21, 0) + +#define V2_QUERY_PF_CAPS_D_NUM_CEQS_S 22 +#define V2_QUERY_PF_CAPS_D_NUM_CEQS_M GENMASK(31, 22) + +#define V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S 0 +#define V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M GENMASK(21, 0) + +#define V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S 22 +#define V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M GENMASK(23, 22) + +#define V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S 24 +#define V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M GENMASK(25, 24) + +#define V2_QUERY_PF_CAPS_D_RSV_PDS_S 0 +#define V2_QUERY_PF_CAPS_D_RSV_PDS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_D_NUM_UARS_S 20 +#define V2_QUERY_PF_CAPS_D_NUM_UARS_M GENMASK(27, 20) + +#define V2_QUERY_PF_CAPS_D_RSV_QPS_S 0 +#define V2_QUERY_PF_CAPS_D_RSV_QPS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_D_RSV_UARS_S 20 +#define V2_QUERY_PF_CAPS_D_RSV_UARS_M GENMASK(27, 20) + +struct hns_roce_query_pf_caps_e { + __le32 chunk_size_shift_rsv_mrws; + __le32 rsv_cqs; + __le32 rsv_srqs; + __le32 rsv_lkey; + __le16 ceq_max_cnt; + __le16 ceq_period; + __le16 aeq_max_cnt; + __le16 aeq_period; +}; + +#define V2_QUERY_PF_CAPS_E_RSV_MRWS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_MRWS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S 20 +#define V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M GENMASK(31, 20) + +#define V2_QUERY_PF_CAPS_E_RSV_CQS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_CQS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_E_RSV_SRQS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_SRQS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_E_RSV_LKEYS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_LKEYS_M GENMASK(19, 0) + struct hns_roce_cmq_desc { __le16 opcode; __le16 flag; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 854ef6e74788..d0031d559213 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -90,7 +90,7 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) { struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); - struct ib_gid_attr zattr = { }; + struct ib_gid_attr zattr = {}; u8 port = attr->port_num - 1; int ret; @@ -210,7 +210,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev, props->max_pkeys = 1; props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay; if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { - props->max_srq = hr_dev->caps.max_srqs; + props->max_srq = hr_dev->caps.num_srqs; props->max_srq_wr = hr_dev->caps.max_srq_wrs; props->max_srq_sge = hr_dev->caps.max_srq_sges; } @@ -259,11 +259,12 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, mtu = iboe_get_mtu(net_dev->mtu); props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256; - props->state = (netif_running(net_dev) && netif_carrier_ok(net_dev)) ? - IB_PORT_ACTIVE : IB_PORT_DOWN; - props->phys_state = (props->state == IB_PORT_ACTIVE) ? - IB_PORT_PHYS_STATE_LINK_UP : - IB_PORT_PHYS_STATE_DISABLED; + props->state = netif_running(net_dev) && netif_carrier_ok(net_dev) ? + IB_PORT_ACTIVE : + IB_PORT_DOWN; + props->phys_state = props->state == IB_PORT_ACTIVE ? + IB_PORT_PHYS_STATE_LINK_UP : + IB_PORT_PHYS_STATE_DISABLED; spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); @@ -481,13 +482,13 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_dev = &hr_dev->ib_dev; - ib_dev->node_type = RDMA_NODE_IB_CA; - ib_dev->dev.parent = dev; + ib_dev->node_type = RDMA_NODE_IB_CA; + ib_dev->dev.parent = dev; - ib_dev->phys_port_cnt = hr_dev->caps.num_ports; - ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey; - ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors; - ib_dev->uverbs_cmd_mask = + ib_dev->phys_port_cnt = hr_dev->caps.num_ports; + ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey; + ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors; + ib_dev->uverbs_cmd_mask = (1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | @@ -503,8 +504,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | (1ULL << IB_USER_VERBS_CMD_DESTROY_QP); - ib_dev->uverbs_ex_cmd_mask |= - (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ); + ib_dev->uverbs_ex_cmd_mask |= (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ); if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR) { ib_dev->uverbs_cmd_mask |= (1ULL << IB_USER_VERBS_CMD_REREG_MR); @@ -589,11 +589,13 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) { ret = hns_roce_init_hem_table(hr_dev, - &hr_dev->mr_table.mtt_cqe_table, - HEM_TYPE_CQE, hr_dev->caps.mtt_entry_sz, - hr_dev->caps.num_cqe_segs, 1); + &hr_dev->mr_table.mtt_cqe_table, + HEM_TYPE_CQE, + hr_dev->caps.mtt_entry_sz, + hr_dev->caps.num_cqe_segs, 1); if (ret) { - dev_err(dev, "Failed to init MTT CQE context memory, aborting.\n"); + dev_err(dev, + "Failed to init CQE context memory, aborting.\n"); goto err_unmap_cqe; } } @@ -633,7 +635,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) hr_dev->caps.num_qps, 1); if (ret) { dev_err(dev, - "Failed to init trrl_table memory, aborting.\n"); + "Failed to init trrl_table memory, aborting.\n"); goto err_unmap_irrl; } } @@ -653,7 +655,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) hr_dev->caps.num_srqs, 1); if (ret) { dev_err(dev, - "Failed to init SRQ context memory, aborting.\n"); + "Failed to init SRQ context memory, aborting.\n"); goto err_unmap_cq; } } @@ -692,33 +694,31 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) hr_dev->caps.num_qps, 1); if (ret) { dev_err(dev, - "Failed to init SCC context memory, aborting.\n"); + "Failed to init SCC context memory, aborting.\n"); goto err_unmap_idx; } } if (hr_dev->caps.qpc_timer_entry_sz) { - ret = hns_roce_init_hem_table(hr_dev, - &hr_dev->qpc_timer_table, + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qpc_timer_table, HEM_TYPE_QPC_TIMER, hr_dev->caps.qpc_timer_entry_sz, hr_dev->caps.num_qpc_timer, 1); if (ret) { dev_err(dev, - "Failed to init QPC timer memory, aborting.\n"); + "Failed to init QPC timer memory, aborting.\n"); goto err_unmap_ctx; } } if (hr_dev->caps.cqc_timer_entry_sz) { - ret = hns_roce_init_hem_table(hr_dev, - &hr_dev->cqc_timer_table, + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cqc_timer_table, HEM_TYPE_CQC_TIMER, hr_dev->caps.cqc_timer_entry_sz, hr_dev->caps.num_cqc_timer, 1); if (ret) { dev_err(dev, - "Failed to init CQC timer memory, aborting.\n"); + "Failed to init CQC timer memory, aborting.\n"); goto err_unmap_qpc_timer; } } @@ -727,8 +727,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) err_unmap_qpc_timer: if (hr_dev->caps.qpc_timer_entry_sz) - hns_roce_cleanup_hem_table(hr_dev, - &hr_dev->qpc_timer_table); + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qpc_timer_table); err_unmap_ctx: if (hr_dev->caps.sccc_entry_sz) @@ -863,6 +862,50 @@ err_uar_table_free: return ret; } +static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq) +{ + struct hns_roce_cq *hr_cq = to_hr_cq(cq); + unsigned long flags; + + spin_lock_irqsave(&hr_cq->lock, flags); + if (cq->comp_handler) { + if (!hr_cq->is_armed) { + hr_cq->is_armed = 1; + list_add_tail(&hr_cq->node, cq_list); + } + } + spin_unlock_irqrestore(&hr_cq->lock, flags); +} + +void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_qp *hr_qp; + struct hns_roce_cq *hr_cq; + struct list_head cq_list; + unsigned long flags_qp; + unsigned long flags; + + INIT_LIST_HEAD(&cq_list); + + spin_lock_irqsave(&hr_dev->qp_list_lock, flags); + list_for_each_entry(hr_qp, &hr_dev->qp_list, node) { + spin_lock_irqsave(&hr_qp->sq.lock, flags_qp); + if (hr_qp->sq.tail != hr_qp->sq.head) + check_and_get_armed_cq(&cq_list, hr_qp->ibqp.send_cq); + spin_unlock_irqrestore(&hr_qp->sq.lock, flags_qp); + + spin_lock_irqsave(&hr_qp->rq.lock, flags_qp); + if ((!hr_qp->ibqp.srq) && (hr_qp->rq.tail != hr_qp->rq.head)) + check_and_get_armed_cq(&cq_list, hr_qp->ibqp.recv_cq); + spin_unlock_irqrestore(&hr_qp->rq.lock, flags_qp); + } + + list_for_each_entry(hr_cq, &cq_list, node) + hns_roce_cq_completion(hr_dev, hr_cq->cqn); + + spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); +} + int hns_roce_init(struct hns_roce_dev *hr_dev) { int ret; @@ -933,6 +976,9 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) } } + INIT_LIST_HEAD(&hr_dev->qp_list); + spin_lock_init(&hr_dev->qp_list_lock); + ret = hns_roce_register_device(hr_dev); if (ret) goto error_failed_register_device; diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 3ff610549c74..b9898e71655a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -1064,8 +1064,8 @@ int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) { if (page_addr & ((1 << mtt->page_shift) - 1)) { dev_err(dev, - "page_addr 0x%llx is not page_shift %d alignment!\n", - page_addr, mtt->page_shift); + "page_addr is not page_shift %d alignment!\n", + mtt->page_shift); ret = -EINVAL; goto out; } diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index eb2ee6a581aa..3257ad11be48 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -393,40 +393,38 @@ static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, /* Get buf size, SQ and RQ are aligned to page_szie */ if (hr_dev->caps.max_sq_sg <= 2) { - hr_qp->buff_size = HNS_ROCE_ALIGN_UP((hr_qp->rq.wqe_cnt << + hr_qp->buff_size = round_up((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), PAGE_SIZE) + - HNS_ROCE_ALIGN_UP((hr_qp->sq.wqe_cnt << + round_up((hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift), PAGE_SIZE); hr_qp->sq.offset = 0; - hr_qp->rq.offset = HNS_ROCE_ALIGN_UP((hr_qp->sq.wqe_cnt << + hr_qp->rq.offset = round_up((hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift), PAGE_SIZE); } else { page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); hr_qp->sge.sge_cnt = ex_sge_num ? max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0; - hr_qp->buff_size = HNS_ROCE_ALIGN_UP((hr_qp->rq.wqe_cnt << + hr_qp->buff_size = round_up((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), page_size) + - HNS_ROCE_ALIGN_UP((hr_qp->sge.sge_cnt << + round_up((hr_qp->sge.sge_cnt << hr_qp->sge.sge_shift), page_size) + - HNS_ROCE_ALIGN_UP((hr_qp->sq.wqe_cnt << + round_up((hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift), page_size); hr_qp->sq.offset = 0; if (ex_sge_num) { - hr_qp->sge.offset = HNS_ROCE_ALIGN_UP( - (hr_qp->sq.wqe_cnt << - hr_qp->sq.wqe_shift), - page_size); + hr_qp->sge.offset = round_up((hr_qp->sq.wqe_cnt << + hr_qp->sq.wqe_shift), + page_size); hr_qp->rq.offset = hr_qp->sge.offset + - HNS_ROCE_ALIGN_UP((hr_qp->sge.sge_cnt << - hr_qp->sge.sge_shift), - page_size); + round_up((hr_qp->sge.sge_cnt << + hr_qp->sge.sge_shift), + page_size); } else { - hr_qp->rq.offset = HNS_ROCE_ALIGN_UP( - (hr_qp->sq.wqe_cnt << - hr_qp->sq.wqe_shift), - page_size); + hr_qp->rq.offset = round_up((hr_qp->sq.wqe_cnt << + hr_qp->sq.wqe_shift), + page_size); } } @@ -593,20 +591,18 @@ static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, /* Get buf size, SQ and RQ are aligned to PAGE_SIZE */ page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); hr_qp->sq.offset = 0; - size = HNS_ROCE_ALIGN_UP(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift, - page_size); + size = round_up(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift, page_size); if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) { hr_qp->sge.sge_cnt = max(page_size/(1 << hr_qp->sge.sge_shift), - (u32)hr_qp->sge.sge_cnt); + (u32)hr_qp->sge.sge_cnt); hr_qp->sge.offset = size; - size += HNS_ROCE_ALIGN_UP(hr_qp->sge.sge_cnt << - hr_qp->sge.sge_shift, page_size); + size += round_up(hr_qp->sge.sge_cnt << hr_qp->sge.sge_shift, + page_size); } hr_qp->rq.offset = size; - size += HNS_ROCE_ALIGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), - page_size); + size += round_up((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), page_size); hr_qp->buff_size = size; /* Get wr and sge number which send */ @@ -681,6 +677,29 @@ static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) kfree(hr_qp->rq_inl_buf.wqe_list); } +static void add_qp_to_list(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct ib_cq *send_cq, struct ib_cq *recv_cq) +{ + struct hns_roce_cq *hr_send_cq, *hr_recv_cq; + unsigned long flags; + + hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; + hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; + + spin_lock_irqsave(&hr_dev->qp_list_lock, flags); + hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); + + list_add_tail(&hr_qp->node, &hr_dev->qp_list); + if (hr_send_cq) + list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); + if (hr_recv_cq) + list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); + + hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); + spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); +} + static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, struct ib_pd *ib_pd, struct ib_qp_init_attr *init_attr, @@ -950,6 +969,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } hr_qp->event = hns_roce_ib_qp_event; + + add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, init_attr->recv_cq); + hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); return 0; @@ -1232,7 +1254,16 @@ out: void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) __acquires(&send_cq->lock) __acquires(&recv_cq->lock) { - if (send_cq == recv_cq) { + if (unlikely(send_cq == NULL && recv_cq == NULL)) { + __acquire(&send_cq->lock); + __acquire(&recv_cq->lock); + } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { + spin_lock_irq(&send_cq->lock); + __acquire(&recv_cq->lock); + } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { + spin_lock_irq(&recv_cq->lock); + __acquire(&send_cq->lock); + } else if (send_cq == recv_cq) { spin_lock_irq(&send_cq->lock); __acquire(&recv_cq->lock); } else if (send_cq->cqn < recv_cq->cqn) { @@ -1248,7 +1279,16 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) __releases(&recv_cq->lock) { - if (send_cq == recv_cq) { + if (unlikely(send_cq == NULL && recv_cq == NULL)) { + __release(&recv_cq->lock); + __release(&send_cq->lock); + } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { + __release(&recv_cq->lock); + spin_unlock(&send_cq->lock); + } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { + __release(&send_cq->lock); + spin_unlock(&recv_cq->lock); + } else if (send_cq == recv_cq) { __release(&recv_cq->lock); spin_unlock_irq(&send_cq->lock); } else if (send_cq->cqn < recv_cq->cqn) { diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c index d44cf33df81a..238614370927 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_main.c +++ b/drivers/infiniband/hw/i40iw/i40iw_main.c @@ -1225,6 +1225,8 @@ static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev) const struct in_ifaddr *ifa; idev = in_dev_get(dev); + if (!idev) + continue; in_dev_for_each_ifa_rtnl(ifa, idev) { i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address, diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c index ecd6cadd529a..b591861934b3 100644 --- a/drivers/infiniband/hw/mlx4/cm.c +++ b/drivers/infiniband/hw/mlx4/cm.c @@ -186,23 +186,6 @@ out: kfree(ent); } -static void id_map_find_del(struct ib_device *ibdev, int pv_cm_id) -{ - struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; - struct rb_root *sl_id_map = &sriov->sl_id_map; - struct id_map_entry *ent, *found_ent; - - spin_lock(&sriov->id_map_lock); - ent = xa_erase(&sriov->pv_id_table, pv_cm_id); - if (!ent) - goto out; - found_ent = id_map_find_by_sl_id(ibdev, ent->slave_id, ent->sl_cm_id); - if (found_ent && found_ent == ent) - rb_erase(&found_ent->node, sl_id_map); -out: - spin_unlock(&sriov->id_map_lock); -} - static void sl_id_map_add(struct ib_device *ibdev, struct id_map_entry *new) { struct rb_root *sl_id_map = &to_mdev(ibdev)->sriov.sl_id_map; @@ -294,7 +277,7 @@ static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id) spin_lock(&sriov->id_map_lock); spin_lock_irqsave(&sriov->going_down_lock, flags); /*make sure that there is no schedule inside the scheduled work.*/ - if (!sriov->is_going_down) { + if (!sriov->is_going_down && !id->scheduled_delete) { id->scheduled_delete = 1; schedule_delayed_work(&id->timeout, CM_CLEANUP_CACHE_TIMEOUT); } @@ -341,9 +324,6 @@ cont: if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) schedule_delayed(ibdev, id); - else if (mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) - id_map_find_del(ibdev, pv_cm_id); - return 0; } @@ -382,12 +362,9 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, *slave = id->slave_id; set_remote_comm_id(mad, id->sl_cm_id); - if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) + if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID || + mad->mad_hdr.attr_id == CM_REJ_ATTR_ID) schedule_delayed(ibdev, id); - else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID || - mad->mad_hdr.attr_id == CM_DREP_ATTR_ID) { - id_map_find_del(ibdev, (int) pv_cm_id); - } return 0; } diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index a57033d4b0e5..f8b936b76dcd 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -568,18 +568,13 @@ static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe, wc->vendor_err = cqe->vendor_err_syndrome; } -static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum) +static int mlx4_ib_ipoib_csum_ok(__be16 status, u8 badfcs_enc, __be16 checksum) { - return ((status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPV4F | - MLX4_CQE_STATUS_IPV4OPT | - MLX4_CQE_STATUS_IPV6 | - MLX4_CQE_STATUS_IPOK)) == - cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPOK)) && - (status & cpu_to_be16(MLX4_CQE_STATUS_UDP | - MLX4_CQE_STATUS_TCP)) && - checksum == cpu_to_be16(0xffff); + return ((badfcs_enc & MLX4_CQE_STATUS_L4_CSUM) || + ((status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && + (status & cpu_to_be16(MLX4_CQE_STATUS_TCP | + MLX4_CQE_STATUS_UDP)) && + (checksum == cpu_to_be16(0xffff)))); } static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, @@ -855,6 +850,7 @@ repoll: wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0; wc->pkey_index = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f; wc->wc_flags |= mlx4_ib_ipoib_csum_ok(cqe->status, + cqe->badfcs_enc, cqe->checksum) ? IB_WC_IP_CSUM_OK : 0; if (is_eth) { wc->slid = 0; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 34055cbab38c..2f5d9b181848 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -246,6 +246,13 @@ static int mlx4_ib_update_gids(struct gid_entry *gids, return mlx4_ib_update_gids_v1(gids, ibdev, port_num); } +static void free_gid_entry(struct gid_entry *entry) +{ + memset(&entry->gid, 0, sizeof(entry->gid)); + kfree(entry->ctx); + entry->ctx = NULL; +} + static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context) { struct mlx4_ib_dev *ibdev = to_mdev(attr->device); @@ -313,6 +320,8 @@ static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context) GFP_ATOMIC); if (!gids) { ret = -ENOMEM; + *context = NULL; + free_gid_entry(&port_gid_table->gids[free]); } else { for (i = 0; i < MLX4_MAX_PORT_GIDS; i++) { memcpy(&gids[i].gid, &port_gid_table->gids[i].gid, sizeof(union ib_gid)); @@ -324,6 +333,12 @@ static int mlx4_ib_add_gid(const struct ib_gid_attr *attr, void **context) if (!ret && hw_update) { ret = mlx4_ib_update_gids(gids, ibdev, attr->port_num); + if (ret) { + spin_lock_bh(&iboe->lock); + *context = NULL; + free_gid_entry(&port_gid_table->gids[free]); + spin_unlock_bh(&iboe->lock); + } kfree(gids); } @@ -353,10 +368,7 @@ static int mlx4_ib_del_gid(const struct ib_gid_attr *attr, void **context) if (!ctx->refcount) { unsigned int real_index = ctx->real_index; - memset(&port_gid_table->gids[real_index].gid, 0, - sizeof(port_gid_table->gids[real_index].gid)); - kfree(port_gid_table->gids[real_index].ctx); - port_gid_table->gids[real_index].ctx = NULL; + free_gid_entry(&port_gid_table->gids[real_index]); hw_update = 1; } } diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index cb23cdb9389a..26425dd2d960 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -849,7 +849,7 @@ static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context, * reused for further WQN allocations. * The next created WQ will allocate a new range. */ - range->dirty = 1; + range->dirty = true; } mutex_unlock(&context->wqn_ranges_mutex); @@ -3085,7 +3085,7 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, const struct ib_ud_wr *wr, } if (ah->av.eth.vlan != cpu_to_be16(0xffff)) { vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff; - is_vlan = 1; + is_vlan = true; } } err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 685b8ed96b4e..d7efc9f6daf0 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -30,7 +30,7 @@ enum devx_obj_flags { struct devx_async_data { struct mlx5_ib_dev *mdev; struct list_head list; - struct ib_uobject *fd_uobj; + struct devx_async_cmd_event_file *ev_file; struct mlx5_async_work cb_work; u16 cmd_out_len; /* must be last field in this structure */ @@ -72,7 +72,6 @@ struct devx_event_subscription { struct rcu_head rcu; u64 cookie; struct devx_async_event_file *ev_file; - struct file *filp; /* Upon hot unplug we need a direct access to */ struct eventfd_ctx *eventfd; }; @@ -1674,21 +1673,20 @@ static void devx_query_callback(int status, struct mlx5_async_work *context) { struct devx_async_data *async_data = container_of(context, struct devx_async_data, cb_work); - struct ib_uobject *fd_uobj = async_data->fd_uobj; - struct devx_async_cmd_event_file *ev_file; - struct devx_async_event_queue *ev_queue; + struct devx_async_cmd_event_file *ev_file = async_data->ev_file; + struct devx_async_event_queue *ev_queue = &ev_file->ev_queue; unsigned long flags; - ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file, - uobj); - ev_queue = &ev_file->ev_queue; - + /* + * Note that if the struct devx_async_cmd_event_file uobj begins to be + * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this + * routine returns, ensuring that it always remains valid here. + */ spin_lock_irqsave(&ev_queue->lock, flags); list_add_tail(&async_data->list, &ev_queue->event_list); spin_unlock_irqrestore(&ev_queue->lock, flags); wake_up_interruptible(&ev_queue->poll_wait); - fput(fd_uobj->object); } #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */ @@ -1757,9 +1755,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( async_data->cmd_out_len = cmd_out_len; async_data->mdev = mdev; - async_data->fd_uobj = fd_uobj; + async_data->ev_file = ev_file; - get_file(fd_uobj->object); MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in, uverbs_attr_get_len(attrs, @@ -1769,12 +1766,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( devx_query_callback, &async_data->cb_work); if (err) - goto cb_err; + goto free_async; return 0; -cb_err: - fput(fd_uobj->object); free_async: kvfree(async_data); sub_bytes: @@ -2032,6 +2027,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( goto err; list_add_tail(&event_sub->event_list, &sub_list); + uverbs_uobject_get(&ev_file->uobj); if (use_eventfd) { event_sub->eventfd = eventfd_ctx_fdget(redirect_fd); @@ -2045,7 +2041,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( event_sub->cookie = cookie; event_sub->ev_file = ev_file; - event_sub->filp = fd_uobj->object; /* May be needed upon cleanup the devx object/subscription */ event_sub->xa_key_level1 = key_level1; event_sub->xa_key_level2 = obj_id; @@ -2099,7 +2094,7 @@ err: if (event_sub->eventfd) eventfd_ctx_put(event_sub->eventfd); - + uverbs_uobject_put(&event_sub->ev_file->uobj); kfree(event_sub); } @@ -2329,6 +2324,9 @@ static int deliver_event(struct devx_event_subscription *event_sub, return 0; } + /* is_destroyed is ignored here because we don't have any memory + * allocation to clean up for the omit_data case + */ list_add_tail(&event_sub->event_list, &ev_file->event_list); spin_unlock_irqrestore(&ev_file->lock, flags); wake_up_interruptible(&ev_file->poll_wait); @@ -2348,7 +2346,10 @@ static int deliver_event(struct devx_event_subscription *event_sub, memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe)); spin_lock_irqsave(&ev_file->lock, flags); - list_add_tail(&event_data->list, &ev_file->event_list); + if (!ev_file->is_destroyed) + list_add_tail(&event_data->list, &ev_file->event_list); + else + kfree(event_data); spin_unlock_irqrestore(&ev_file->lock, flags); wake_up_interruptible(&ev_file->poll_wait); @@ -2361,17 +2362,10 @@ static void dispatch_event_fd(struct list_head *fd_list, struct devx_event_subscription *item; list_for_each_entry_rcu(item, fd_list, xa_list) { - if (!get_file_rcu(item->filp)) - continue; - - if (item->eventfd) { + if (item->eventfd) eventfd_signal(item->eventfd, 1); - fput(item->filp); - continue; - } - - deliver_event(item, data); - fput(item->filp); + else + deliver_event(item, data); } } @@ -2509,23 +2503,6 @@ static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, return ret; } -static int devx_async_cmd_event_close(struct inode *inode, struct file *filp) -{ - struct ib_uobject *uobj = filp->private_data; - struct devx_async_cmd_event_file *comp_ev_file = container_of( - uobj, struct devx_async_cmd_event_file, uobj); - struct devx_async_data *entry, *tmp; - - spin_lock_irq(&comp_ev_file->ev_queue.lock); - list_for_each_entry_safe(entry, tmp, - &comp_ev_file->ev_queue.event_list, list) - kvfree(entry); - spin_unlock_irq(&comp_ev_file->ev_queue.lock); - - uverbs_close_fd(filp); - return 0; -} - static __poll_t devx_async_cmd_event_poll(struct file *filp, struct poll_table_struct *wait) { @@ -2549,7 +2526,7 @@ static const struct file_operations devx_async_cmd_event_fops = { .owner = THIS_MODULE, .read = devx_async_cmd_event_read, .poll = devx_async_cmd_event_poll, - .release = devx_async_cmd_event_close, + .release = uverbs_uobject_fd_release, .llseek = no_llseek, }; @@ -2653,81 +2630,85 @@ static __poll_t devx_async_event_poll(struct file *filp, return pollflags; } -static int devx_async_event_close(struct inode *inode, struct file *filp) +static void devx_free_subscription(struct rcu_head *rcu) { - struct devx_async_event_file *ev_file = filp->private_data; - struct devx_event_subscription *event_sub, *event_sub_tmp; - struct devx_async_event_data *entry, *tmp; - struct mlx5_ib_dev *dev = ev_file->dev; - - mutex_lock(&dev->devx_event_table.event_xa_lock); - /* delete the subscriptions which are related to this FD */ - list_for_each_entry_safe(event_sub, event_sub_tmp, - &ev_file->subscribed_events_list, file_list) { - devx_cleanup_subscription(dev, event_sub); - if (event_sub->eventfd) - eventfd_ctx_put(event_sub->eventfd); - - list_del_rcu(&event_sub->file_list); - /* subscription may not be used by the read API any more */ - kfree_rcu(event_sub, rcu); - } - - mutex_unlock(&dev->devx_event_table.event_xa_lock); + struct devx_event_subscription *event_sub = + container_of(rcu, struct devx_event_subscription, rcu); - /* free the pending events allocation */ - if (!ev_file->omit_data) { - spin_lock_irq(&ev_file->lock); - list_for_each_entry_safe(entry, tmp, - &ev_file->event_list, list) - kfree(entry); /* read can't come any more */ - spin_unlock_irq(&ev_file->lock); - } - - uverbs_close_fd(filp); - put_device(&dev->ib_dev.dev); - return 0; + if (event_sub->eventfd) + eventfd_ctx_put(event_sub->eventfd); + uverbs_uobject_put(&event_sub->ev_file->uobj); + kfree(event_sub); } static const struct file_operations devx_async_event_fops = { .owner = THIS_MODULE, .read = devx_async_event_read, .poll = devx_async_event_poll, - .release = devx_async_event_close, + .release = uverbs_uobject_fd_release, .llseek = no_llseek, }; -static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, - enum rdma_remove_reason why) +static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) { struct devx_async_cmd_event_file *comp_ev_file = container_of(uobj, struct devx_async_cmd_event_file, uobj); struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; + struct devx_async_data *entry, *tmp; spin_lock_irq(&ev_queue->lock); ev_queue->is_destroyed = 1; spin_unlock_irq(&ev_queue->lock); - - if (why == RDMA_REMOVE_DRIVER_REMOVE) - wake_up_interruptible(&ev_queue->poll_wait); + wake_up_interruptible(&ev_queue->poll_wait); mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx); + + spin_lock_irq(&comp_ev_file->ev_queue.lock); + list_for_each_entry_safe(entry, tmp, + &comp_ev_file->ev_queue.event_list, list) + kvfree(entry); + spin_unlock_irq(&comp_ev_file->ev_queue.lock); return 0; }; -static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj, - enum rdma_remove_reason why) +static int devx_async_event_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) { struct devx_async_event_file *ev_file = container_of(uobj, struct devx_async_event_file, uobj); + struct devx_event_subscription *event_sub, *event_sub_tmp; + struct devx_async_event_data *entry, *tmp; + struct mlx5_ib_dev *dev = ev_file->dev; spin_lock_irq(&ev_file->lock); ev_file->is_destroyed = 1; spin_unlock_irq(&ev_file->lock); - wake_up_interruptible(&ev_file->poll_wait); + + mutex_lock(&dev->devx_event_table.event_xa_lock); + /* delete the subscriptions which are related to this FD */ + list_for_each_entry_safe(event_sub, event_sub_tmp, + &ev_file->subscribed_events_list, file_list) { + devx_cleanup_subscription(dev, event_sub); + list_del_rcu(&event_sub->file_list); + /* subscription may not be used by the read API any more */ + call_rcu(&event_sub->rcu, devx_free_subscription); + } + mutex_unlock(&dev->devx_event_table.event_xa_lock); + + /* free the pending events allocation */ + if (!ev_file->omit_data) { + spin_lock_irq(&ev_file->lock); + list_for_each_entry_safe(entry, tmp, + &ev_file->event_list, list) + kfree(entry); /* read can't come any more */ + spin_unlock_irq(&ev_file->lock); + } + + put_device(&dev->ib_dev.dev); return 0; }; @@ -2913,7 +2894,7 @@ DECLARE_UVERBS_NAMED_METHOD( DECLARE_UVERBS_NAMED_OBJECT( MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file), - devx_hot_unplug_async_cmd_event_file, + devx_async_cmd_event_destroy_uobj, &devx_async_cmd_event_fops, "[devx_async_cmd]", O_RDONLY), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)); @@ -2931,7 +2912,7 @@ DECLARE_UVERBS_NAMED_METHOD( DECLARE_UVERBS_NAMED_OBJECT( MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file), - devx_hot_unplug_async_event_file, + devx_async_event_destroy_uobj, &devx_async_event_fops, "[devx_async_event]", O_RDONLY), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)); diff --git a/drivers/infiniband/hw/mlx5/gsi.c b/drivers/infiniband/hw/mlx5/gsi.c index ac4d8d1b9a07..1ae6fd95acaa 100644 --- a/drivers/infiniband/hw/mlx5/gsi.c +++ b/drivers/infiniband/hw/mlx5/gsi.c @@ -507,8 +507,7 @@ int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr, ret = ib_post_send(tx_qp, &cur_wr.wr, bad_wr); if (ret) { /* Undo the effect of adding the outstanding wr */ - gsi->outstanding_pi = (gsi->outstanding_pi - 1) % - gsi->cap.max_send_wr; + gsi->outstanding_pi--; goto err; } spin_unlock_irqrestore(&gsi->lock, flags); diff --git a/drivers/infiniband/hw/mlx5/ib_virt.c b/drivers/infiniband/hw/mlx5/ib_virt.c index 4f0edd4832bd..b61165359954 100644 --- a/drivers/infiniband/hw/mlx5/ib_virt.c +++ b/drivers/infiniband/hw/mlx5/ib_virt.c @@ -164,8 +164,10 @@ static int set_vf_node_guid(struct ib_device *device, int vf, u8 port, u64 guid) in->field_select = MLX5_HCA_VPORT_SEL_NODE_GUID; in->node_guid = guid; err = mlx5_core_modify_hca_vport_context(mdev, 1, 1, vf + 1, in); - if (!err) + if (!err) { vfs_ctx[vf].node_guid = guid; + vfs_ctx[vf].node_guid_valid = 1; + } kfree(in); return err; } @@ -185,8 +187,10 @@ static int set_vf_port_guid(struct ib_device *device, int vf, u8 port, u64 guid) in->field_select = MLX5_HCA_VPORT_SEL_PORT_GUID; in->port_guid = guid; err = mlx5_core_modify_hca_vport_context(mdev, 1, 1, vf + 1, in); - if (!err) + if (!err) { vfs_ctx[vf].port_guid = guid; + vfs_ctx[vf].port_guid_valid = 1; + } kfree(in); return err; } @@ -208,20 +212,12 @@ int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u8 port, { struct mlx5_ib_dev *dev = to_mdev(device); struct mlx5_core_dev *mdev = dev->mdev; - struct mlx5_hca_vport_context *rep; - int err; - - rep = kzalloc(sizeof(*rep), GFP_KERNEL); - if (!rep) - return -ENOMEM; + struct mlx5_vf_context *vfs_ctx = mdev->priv.sriov.vfs_ctx; - err = mlx5_query_hca_vport_context(mdev, 1, 1, vf+1, rep); - if (err) - goto ex; + node_guid->guid = + vfs_ctx[vf].node_guid_valid ? vfs_ctx[vf].node_guid : 0; + port_guid->guid = + vfs_ctx[vf].port_guid_valid ? vfs_ctx[vf].port_guid : 0; - port_guid->guid = rep->port_guid; - node_guid->guid = rep->node_guid; -ex: - kfree(rep); - return err; + return 0; } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index ab4ec33409b8..e874d688d040 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2094,6 +2094,7 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry) { struct mlx5_user_mmap_entry *mentry = to_mmmap(entry); struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device); + struct mlx5_var_table *var_table = &dev->var_table; struct mlx5_ib_dm *mdm; switch (mentry->mmap_flag) { @@ -2103,6 +2104,12 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry) mdm->size); kfree(mdm); break; + case MLX5_IB_MMAP_TYPE_VAR: + mutex_lock(&var_table->bitmap_lock); + clear_bit(mentry->page_idx, var_table->bitmap); + mutex_unlock(&var_table->bitmap_lock); + kfree(mentry); + break; default: WARN_ON(true); } @@ -2262,7 +2269,10 @@ static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev, mentry = to_mmmap(entry); pfn = (mentry->address >> PAGE_SHIFT); - prot = pgprot_writecombine(vma->vm_page_prot); + if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR) + prot = pgprot_noncached(vma->vm_page_prot); + else + prot = pgprot_writecombine(vma->vm_page_prot); ret = rdma_user_mmap_io(ucontext, vma, pfn, entry->npages * PAGE_SIZE, prot, @@ -2271,6 +2281,15 @@ static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev, return ret; } +static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry) +{ + u16 cmd = entry->rdma_entry.start_pgoff >> 16; + u16 index = entry->rdma_entry.start_pgoff & 0xFFFF; + + return (((index >> 8) << 16) | (cmd << MLX5_IB_MMAP_CMD_SHIFT) | + (index & 0xFF)) << PAGE_SHIFT; +} + static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) { struct mlx5_ib_ucontext *context = to_mucontext(ibcontext); @@ -5368,6 +5387,14 @@ static const struct mlx5_ib_counter extended_err_cnts[] = { INIT_Q_COUNTER(req_cqe_flush_error), }; +static const struct mlx5_ib_counter roce_accl_cnts[] = { + INIT_Q_COUNTER(roce_adp_retrans), + INIT_Q_COUNTER(roce_adp_retrans_to), + INIT_Q_COUNTER(roce_slow_restart), + INIT_Q_COUNTER(roce_slow_restart_cnps), + INIT_Q_COUNTER(roce_slow_restart_trans), +}; + #define INIT_EXT_PPCNT_COUNTER(_name) \ { .name = #_name, .offset = \ MLX5_BYTE_OFF(ppcnt_reg, \ @@ -5416,6 +5443,9 @@ static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) num_counters += ARRAY_SIZE(extended_err_cnts); + if (MLX5_CAP_GEN(dev->mdev, roce_accl)) + num_counters += ARRAY_SIZE(roce_accl_cnts); + cnts->num_q_counters = num_counters; if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { @@ -5476,6 +5506,13 @@ static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, } } + if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { + for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) { + names[j] = roce_accl_cnts[i].name; + offsets[j] = roce_accl_cnts[i].offset; + } + } + if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { names[j] = cong_cnts[i].name; @@ -6051,6 +6088,145 @@ static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev) mlx5_nic_vport_disable_roce(dev->mdev); } +static int var_obj_cleanup(struct ib_uobject *uobject, + enum rdma_remove_reason why, + struct uverbs_attr_bundle *attrs) +{ + struct mlx5_user_mmap_entry *obj = uobject->object; + + rdma_user_mmap_entry_remove(&obj->rdma_entry); + return 0; +} + +static struct mlx5_user_mmap_entry * +alloc_var_entry(struct mlx5_ib_ucontext *c) +{ + struct mlx5_user_mmap_entry *entry; + struct mlx5_var_table *var_table; + u32 page_idx; + int err; + + var_table = &to_mdev(c->ibucontext.device)->var_table; + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return ERR_PTR(-ENOMEM); + + mutex_lock(&var_table->bitmap_lock); + page_idx = find_first_zero_bit(var_table->bitmap, + var_table->num_var_hw_entries); + if (page_idx >= var_table->num_var_hw_entries) { + err = -ENOSPC; + mutex_unlock(&var_table->bitmap_lock); + goto end; + } + + set_bit(page_idx, var_table->bitmap); + mutex_unlock(&var_table->bitmap_lock); + + entry->address = var_table->hw_start_addr + + (page_idx * var_table->stride_size); + entry->page_idx = page_idx; + entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR; + + err = rdma_user_mmap_entry_insert_range( + &c->ibucontext, &entry->rdma_entry, var_table->stride_size, + MLX5_IB_MMAP_OFFSET_START << 16, + (MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1); + if (err) + goto err_insert; + + return entry; + +err_insert: + mutex_lock(&var_table->bitmap_lock); + clear_bit(page_idx, var_table->bitmap); + mutex_unlock(&var_table->bitmap_lock); +end: + kfree(entry); + return ERR_PTR(err); +} + +static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = uverbs_attr_get_uobject( + attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE); + struct mlx5_ib_ucontext *c; + struct mlx5_user_mmap_entry *entry; + u64 mmap_offset; + u32 length; + int err; + + c = to_mucontext(ib_uverbs_get_ucontext(attrs)); + if (IS_ERR(c)) + return PTR_ERR(c); + + entry = alloc_var_entry(c); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + mmap_offset = mlx5_entry_to_mmap_offset(entry); + length = entry->rdma_entry.npages * PAGE_SIZE; + uobj->object = entry; + + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET, + &mmap_offset, sizeof(mmap_offset)); + if (err) + goto err; + + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID, + &entry->page_idx, sizeof(entry->page_idx)); + if (err) + goto err; + + err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH, + &length, sizeof(length)); + if (err) + goto err; + + return 0; + +err: + rdma_user_mmap_entry_remove(&entry->rdma_entry); + return err; +} + +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_VAR_OBJ_ALLOC, + UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE, + MLX5_IB_OBJECT_VAR, + UVERBS_ACCESS_NEW, + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID, + UVERBS_ATTR_TYPE(u32), + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH, + UVERBS_ATTR_TYPE(u32), + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET, + UVERBS_ATTR_TYPE(u64), + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD_DESTROY( + MLX5_IB_METHOD_VAR_OBJ_DESTROY, + UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE, + MLX5_IB_OBJECT_VAR, + UVERBS_ACCESS_DESTROY, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR, + UVERBS_TYPE_ALLOC_IDR(var_obj_cleanup), + &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC), + &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY)); + +static bool var_is_supported(struct ib_device *device) +{ + struct mlx5_ib_dev *dev = to_mdev(device); + + return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) & + MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q); +} + ADD_UVERBS_ATTRIBUTES_SIMPLE( mlx5_ib_dm, UVERBS_OBJECT_DM, @@ -6073,14 +6249,14 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE( enum mlx5_ib_uapi_flow_action_flags)); static const struct uapi_definition mlx5_ib_defs[] = { -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) UAPI_DEF_CHAIN(mlx5_ib_devx_defs), UAPI_DEF_CHAIN(mlx5_ib_flow_defs), -#endif UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, &mlx5_ib_flow_action), UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm), + UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR, + UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)), {} }; @@ -6352,6 +6528,35 @@ static const struct ib_device_ops mlx5_ib_dev_dm_ops = { .reg_dm_mr = mlx5_ib_reg_dm_mr, }; +static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev) +{ + struct mlx5_core_dev *mdev = dev->mdev; + struct mlx5_var_table *var_table = &dev->var_table; + u8 log_doorbell_bar_size; + u8 log_doorbell_stride; + u64 bar_size; + + log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev, + log_doorbell_bar_size); + log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev, + log_doorbell_stride); + var_table->hw_start_addr = dev->mdev->bar_addr + + MLX5_CAP64_DEV_VDPA_EMULATION(mdev, + doorbell_bar_offset); + bar_size = (1ULL << log_doorbell_bar_size) * 4096; + var_table->stride_size = 1ULL << log_doorbell_stride; + var_table->num_var_hw_entries = bar_size / var_table->stride_size; + mutex_init(&var_table->bitmap_lock); + var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries, + GFP_KERNEL); + return (var_table->bitmap) ? 0 : -ENOMEM; +} + +static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev) +{ + bitmap_free(dev->var_table.bitmap); +} + static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) { struct mlx5_core_dev *mdev = dev->mdev; @@ -6439,6 +6644,13 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) mutex_init(&dev->lb.mutex); + if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) & + MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) { + err = mlx5_ib_init_var_table(dev); + if (err) + return err; + } + dev->ib_dev.use_cq_dim = true; return 0; @@ -6742,6 +6954,8 @@ void __mlx5_ib_remove(struct mlx5_ib_dev *dev, const struct mlx5_ib_profile *profile, int stage) { + dev->ib_active = false; + /* Number of stages to cleanup */ while (stage) { stage--; @@ -6787,7 +7001,7 @@ static const struct mlx5_ib_profile pf_profile = { mlx5_ib_stage_flow_db_cleanup), STAGE_CREATE(MLX5_IB_STAGE_CAPS, mlx5_ib_stage_caps_init, - NULL), + mlx5_ib_stage_caps_cleanup), STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB, mlx5_ib_stage_non_default_cb, NULL), @@ -6844,7 +7058,7 @@ const struct mlx5_ib_profile raw_eth_profile = { mlx5_ib_stage_flow_db_cleanup), STAGE_CREATE(MLX5_IB_STAGE_CAPS, mlx5_ib_stage_caps_init, - NULL), + mlx5_ib_stage_caps_cleanup), STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB, mlx5_ib_stage_raw_eth_non_default_cb, NULL), diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c index 048f4e974a61..b90a3649e7d1 100644 --- a/drivers/infiniband/hw/mlx5/mem.c +++ b/drivers/infiniband/hw/mlx5/mem.c @@ -101,18 +101,6 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, *count = i; } -static u64 umem_dma_to_mtt(dma_addr_t umem_dma) -{ - u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK; - - if (umem_dma & ODP_READ_ALLOWED_BIT) - mtt_entry |= MLX5_IB_MTT_READ; - if (umem_dma & ODP_WRITE_ALLOWED_BIT) - mtt_entry |= MLX5_IB_MTT_WRITE; - - return mtt_entry; -} - /* * Populate the given array with bus addresses from the umem. * @@ -139,19 +127,6 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, struct scatterlist *sg; int entry; - if (umem->is_odp) { - WARN_ON(shift != 0); - WARN_ON(access_flags != (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)); - - for (i = 0; i < num_pages; ++i) { - dma_addr_t pa = - to_ib_umem_odp(umem)->dma_list[offset + i]; - - pas[i] = cpu_to_be64(umem_dma_to_mtt(pa)); - } - return; - } - i = 0; for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { len = sg_dma_len(sg) >> PAGE_SHIFT; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 77d495b2032d..d9bffcc93587 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -72,6 +72,11 @@ #define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size) enum { + MLX5_IB_MMAP_OFFSET_START = 9, + MLX5_IB_MMAP_OFFSET_END = 255, +}; + +enum { MLX5_IB_MMAP_CMD_SHIFT = 8, MLX5_IB_MMAP_CMD_MASK = 0xff, }; @@ -120,6 +125,7 @@ enum { enum mlx5_ib_mmap_type { MLX5_IB_MMAP_TYPE_MEMIC = 1, + MLX5_IB_MMAP_TYPE_VAR = 2, }; #define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev) \ @@ -563,6 +569,7 @@ struct mlx5_user_mmap_entry { struct rdma_user_mmap_entry rdma_entry; u8 mmap_flag; u64 address; + u32 page_idx; }; struct mlx5_ib_dm { @@ -959,6 +966,15 @@ struct mlx5_devx_event_table { struct xarray event_xa; }; +struct mlx5_var_table { + /* serialize updating the bitmap */ + struct mutex bitmap_lock; + unsigned long *bitmap; + u64 hw_start_addr; + u32 stride_size; + u64 num_var_hw_entries; +}; + struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; @@ -1013,6 +1029,7 @@ struct mlx5_ib_dev { struct mlx5_srq_table srq_table; struct mlx5_async_ctx async_ctx; struct mlx5_devx_event_table devx_event_table; + struct mlx5_var_table var_table; struct xarray sig_mrs; }; @@ -1276,8 +1293,8 @@ void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev); int __init mlx5_ib_odp_init(void); void mlx5_ib_odp_cleanup(void); void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent); -void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, - size_t nentries, struct mlx5_ib_mr *mr, int flags); +void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, + struct mlx5_ib_mr *mr, int flags); int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, @@ -1293,9 +1310,8 @@ static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {} static inline int mlx5_ib_odp_init(void) { return 0; } static inline void mlx5_ib_odp_cleanup(void) {} static inline void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) {} -static inline void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, - size_t nentries, struct mlx5_ib_mr *mr, - int flags) {} +static inline void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, + struct mlx5_ib_mr *mr, int flags) {} static inline int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, @@ -1363,14 +1379,14 @@ int mlx5_ib_fill_res_entry(struct sk_buff *msg, int mlx5_ib_fill_stat_entry(struct sk_buff *msg, struct rdma_restrack_entry *res); +extern const struct uapi_definition mlx5_ib_devx_defs[]; +extern const struct uapi_definition mlx5_ib_flow_defs[]; + #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user); void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid); void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev); void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev); -const struct uverbs_object_tree_def *mlx5_ib_get_devx_tree(void); -extern const struct uapi_definition mlx5_ib_devx_defs[]; -extern const struct uapi_definition mlx5_ib_flow_defs[]; struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add( struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, struct mlx5_flow_context *flow_context, @@ -1378,7 +1394,6 @@ struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add( void *cmd_in, int inlen, int dest_id, int dest_type); bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type); bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id); -int mlx5_ib_get_flow_trees(const struct uverbs_object_tree_def **root); void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction); #else static inline int @@ -1507,7 +1522,7 @@ int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter); u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num); static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, - bool do_modify_atomic) + bool do_modify_atomic, int access_flags) { if (MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) return false; @@ -1517,6 +1532,9 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)) return false; + if (access_flags & IB_ACCESS_RELAXED_ORDERING) + return false; + return true; } diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 44a0ee6bd9f1..6fa0a83c19de 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -147,7 +147,7 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num) break; } mr->order = ent->order; - mr->allocated_from_cache = 1; + mr->allocated_from_cache = true; mr->dev = dev; MLX5_SET(mkc, mkc, free, 1); @@ -661,12 +661,21 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, struct ib_pd *pd) { + struct mlx5_ib_dev *dev = to_mdev(pd->device); + MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); MLX5_SET(mkc, mkc, lr, 1); + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) + MLX5_SET(mkc, mkc, relaxed_ordering_write, + !!(acc & IB_ACCESS_RELAXED_ORDERING)); + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) + MLX5_SET(mkc, mkc, relaxed_ordering_read, + !!(acc & IB_ACCESS_RELAXED_ORDERING)); + MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET64(mkc, mkc, start_addr, start_addr); @@ -867,36 +876,6 @@ static struct mlx5_ib_mr *alloc_mr_from_cache( return mr; } -static inline int populate_xlt(struct mlx5_ib_mr *mr, int idx, int npages, - void *xlt, int page_shift, size_t size, - int flags) -{ - struct mlx5_ib_dev *dev = mr->dev; - struct ib_umem *umem = mr->umem; - - if (flags & MLX5_IB_UPD_XLT_INDIRECT) { - if (!umr_can_use_indirect_mkey(dev)) - return -EPERM; - mlx5_odp_populate_klm(xlt, idx, npages, mr, flags); - return npages; - } - - npages = min_t(size_t, npages, ib_umem_num_pages(umem) - idx); - - if (!(flags & MLX5_IB_UPD_XLT_ZAP)) { - __mlx5_ib_populate_pas(dev, umem, page_shift, - idx, npages, xlt, - MLX5_IB_MTT_PRESENT); - /* Clear padding after the pages - * brought from the umem. - */ - memset(xlt + (npages * sizeof(struct mlx5_mtt)), 0, - size - npages * sizeof(struct mlx5_mtt)); - } - - return npages; -} - #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \ MLX5_UMR_MTT_ALIGNMENT) #define MLX5_SPARE_UMR_CHUNK 0x10000 @@ -920,6 +899,7 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, size_t pages_mapped = 0; size_t pages_to_map = 0; size_t pages_iter = 0; + size_t size_to_map = 0; gfp_t gfp; bool use_emergency_page = false; @@ -966,6 +946,15 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, goto free_xlt; } + if (mr->umem->is_odp) { + if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) { + struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); + size_t max_pages = ib_umem_odp_num_pages(odp) - idx; + + pages_to_map = min_t(size_t, pages_to_map, max_pages); + } + } + sg.addr = dma; sg.lkey = dev->umrc.pd->local_dma_lkey; @@ -988,14 +977,22 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, pages_mapped < pages_to_map && !err; pages_mapped += pages_iter, idx += pages_iter) { npages = min_t(int, pages_iter, pages_to_map - pages_mapped); + size_to_map = npages * desc_size; dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE); - npages = populate_xlt(mr, idx, npages, xlt, - page_shift, size, flags); - + if (mr->umem->is_odp) { + mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); + } else { + __mlx5_ib_populate_pas(dev, mr->umem, page_shift, idx, + npages, xlt, + MLX5_IB_MTT_PRESENT); + /* Clear padding after the pages + * brought from the umem. + */ + memset(xlt + size_to_map, 0, size - size_to_map); + } dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE); - sg.length = ALIGN(npages * desc_size, - MLX5_UMR_MTT_ALIGNMENT); + sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT); if (pages_mapped + pages_iter >= pages_to_map) { if (flags & MLX5_IB_UPD_XLT_ENABLE) @@ -1074,6 +1071,12 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) + MLX5_SET(mkc, mkc, relaxed_ordering_write, + !!(access_flags & IB_ACCESS_RELAXED_ORDERING)); + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) + MLX5_SET(mkc, mkc, relaxed_ordering_read, + !!(access_flags & IB_ACCESS_RELAXED_ORDERING)); MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE)); MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ)); @@ -1264,7 +1267,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (err < 0) return ERR_PTR(err); - use_umr = mlx5_ib_can_use_umr(dev, true); + use_umr = mlx5_ib_can_use_umr(dev, true, access_flags); if (order <= mr_cache_max_order(dev) && use_umr) { mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont, @@ -1431,7 +1434,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, goto err; } - if (!mlx5_ib_can_use_umr(dev, true) || + if (!mlx5_ib_can_use_umr(dev, true, access_flags) || (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len))) { /* * UMR can't be used - MKey needs to be replaced. @@ -1452,7 +1455,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, goto err; } - mr->allocated_from_cache = 0; + mr->allocated_from_cache = false; } else { /* * Send a UMR WQE diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 0afb0042bd53..4216814ba871 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -93,8 +93,8 @@ struct mlx5_pagefault { static u64 mlx5_imr_ksm_entries; -void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, - struct mlx5_ib_mr *imr, int flags) +static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, + struct mlx5_ib_mr *imr, int flags) { struct mlx5_klm *end = pklm + nentries; @@ -144,6 +144,44 @@ void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, } } +static u64 umem_dma_to_mtt(dma_addr_t umem_dma) +{ + u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK; + + if (umem_dma & ODP_READ_ALLOWED_BIT) + mtt_entry |= MLX5_IB_MTT_READ; + if (umem_dma & ODP_WRITE_ALLOWED_BIT) + mtt_entry |= MLX5_IB_MTT_WRITE; + + return mtt_entry; +} + +static void populate_mtt(__be64 *pas, size_t idx, size_t nentries, + struct mlx5_ib_mr *mr, int flags) +{ + struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); + dma_addr_t pa; + size_t i; + + if (flags & MLX5_IB_UPD_XLT_ZAP) + return; + + for (i = 0; i < nentries; i++) { + pa = odp->dma_list[idx + i]; + pas[i] = cpu_to_be64(umem_dma_to_mtt(pa)); + } +} + +void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, + struct mlx5_ib_mr *mr, int flags) +{ + if (flags & MLX5_IB_UPD_XLT_INDIRECT) { + populate_klm(xlt, idx, nentries, mr, flags); + } else { + populate_mtt(xlt, idx, nentries, mr, flags); + } +} + static void dma_fence_odp_mr(struct mlx5_ib_mr *mr) { struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); @@ -342,7 +380,7 @@ void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) memset(caps, 0, sizeof(*caps)); if (!MLX5_CAP_GEN(dev->mdev, pg) || - !mlx5_ib_can_use_umr(dev, true)) + !mlx5_ib_can_use_umr(dev, true, 0)) return; caps->general_caps = IB_ODP_SUPPORT; diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index ae7cbd9c9bca..a4f8e7030787 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -1918,7 +1918,7 @@ static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev, { enum ib_qp_type qpt = init_attr->qp_type; int scqe_sz; - bool allow_scat_cqe = 0; + bool allow_scat_cqe = false; if (qpt == IB_QPT_UC || qpt == IB_QPT_UD) return; @@ -4870,7 +4870,7 @@ static int set_reg_wr(struct mlx5_ib_qp *qp, bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC; u8 flags = 0; - if (!mlx5_ib_can_use_umr(dev, atomic)) { + if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) { mlx5_ib_warn(to_mdev(qp->ibqp.device), "Fast update of %s for MR is disabled\n", (MLX5_CAP_GEN(dev->mdev, diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 920f35e28cfc..484b555150e0 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -312,7 +312,18 @@ int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) } ctx->db_mmap_entry = &entry->rdma_entry; - uresp.dpm_enabled = dev->user_dpm_enabled; + if (!dev->user_dpm_enabled) + uresp.dpm_flags = 0; + else if (rdma_protocol_iwarp(&dev->ibdev, 1)) + uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY; + else + uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED | + QEDR_DPM_TYPE_ROCE_LEGACY; + + uresp.dpm_flags |= QEDR_DPM_SIZES_SET; + uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; + uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; + uresp.wids_enabled = 1; uresp.wid_count = oparams.wid_count; uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry); diff --git a/drivers/infiniband/sw/rdmavt/rc.c b/drivers/infiniband/sw/rdmavt/rc.c index 890d7b760d2e..977906cc0d11 100644 --- a/drivers/infiniband/sw/rdmavt/rc.c +++ b/drivers/infiniband/sw/rdmavt/rc.c @@ -195,7 +195,14 @@ void rvt_get_credit(struct rvt_qp *qp, u32 aeth) } EXPORT_SYMBOL(rvt_get_credit); -/* rvt_restart_sge - rewind the sge state for a wqe */ +/** + * rvt_restart_sge - rewind the sge state for a wqe + * @ss: the sge state pointer + * @wqe: the wqe to rewind + * @len: the data length from the start of the wqe in bytes + * + * Returns the remaining data length. + */ u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len) { ss->sge = wqe->sg_list[0]; diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h index 353c6668249e..f59616b02477 100644 --- a/drivers/infiniband/sw/rxe/rxe_param.h +++ b/drivers/infiniband/sw/rxe/rxe_param.h @@ -34,6 +34,8 @@ #ifndef RXE_PARAM_H #define RXE_PARAM_H +#include <uapi/rdma/rdma_user_rxe.h> + static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu) { if (mtu < 256) @@ -64,7 +66,6 @@ enum rxe_device_param { RXE_PAGE_SIZE_CAP = 0xfffff000, RXE_MAX_QP = 0x10000, RXE_MAX_QP_WR = 0x4000, - RXE_MAX_INLINE_DATA = 400, RXE_DEVICE_CAP_FLAGS = IB_DEVICE_BAD_PKEY_CNTR | IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_AUTO_PATH_MIG @@ -77,6 +78,10 @@ enum rxe_device_param { | IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_ALLOW_USER_UNREG, RXE_MAX_SGE = 32, + RXE_MAX_WQE_SIZE = sizeof(struct rxe_send_wqe) + + sizeof(struct ib_sge) * RXE_MAX_SGE, + RXE_MAX_INLINE_DATA = RXE_MAX_WQE_SIZE - + sizeof(struct rxe_send_wqe), RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 16384, RXE_MAX_LOG_CQE = 15, diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index e2c6d1cedf41..ec21f616ac98 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -237,19 +237,17 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, */ qp->src_port = RXE_ROCE_V2_SPORT + (hash_32_generic(qp_num(qp), 14) & 0x3fff); - qp->sq.max_wr = init->cap.max_send_wr; - qp->sq.max_sge = init->cap.max_send_sge; - qp->sq.max_inline = init->cap.max_inline_data; - wqe_size = max_t(int, sizeof(struct rxe_send_wqe) + - qp->sq.max_sge * sizeof(struct ib_sge), - sizeof(struct rxe_send_wqe) + - qp->sq.max_inline); + /* These caps are limited by rxe_qp_chk_cap() done by the caller */ + wqe_size = max_t(int, init->cap.max_send_sge * sizeof(struct ib_sge), + init->cap.max_inline_data); + qp->sq.max_sge = init->cap.max_send_sge = + wqe_size / sizeof(struct ib_sge); + qp->sq.max_inline = init->cap.max_inline_data = wqe_size; + wqe_size += sizeof(struct rxe_send_wqe); - qp->sq.queue = rxe_queue_init(rxe, - &qp->sq.max_wr, - wqe_size); + qp->sq.queue = rxe_queue_init(rxe, &qp->sq.max_wr, wqe_size); if (!qp->sq.queue) return -ENOMEM; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index 95834206c80c..92de39c4a7c1 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -408,7 +408,7 @@ struct rxe_dev { struct list_head pending_mmaps; spinlock_t mmap_offset_lock; /* guard mmap_offset */ - int mmap_offset; + u64 mmap_offset; atomic64_t stats_counters[RXE_NUM_OF_COUNTERS]; diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h index b939f489cd46..af5e9f8c0fcd 100644 --- a/drivers/infiniband/sw/siw/siw.h +++ b/drivers/infiniband/sw/siw/siw.h @@ -7,6 +7,7 @@ #define _SIW_H #include <rdma/ib_verbs.h> +#include <rdma/restrack.h> #include <linux/socket.h> #include <linux/skbuff.h> #include <crypto/hash.h> @@ -209,7 +210,6 @@ struct siw_cq { u32 cq_put; u32 cq_get; u32 num_cqe; - bool kernel_verbs; struct rdma_user_mmap_entry *cq_entry; /* mmap info for CQE array */ u32 id; /* For debugging only */ }; @@ -254,8 +254,8 @@ struct siw_srq { u32 rq_get; u32 num_rqe; /* max # of wqe's allowed */ struct rdma_user_mmap_entry *srq_entry; /* mmap info for SRQ array */ - char armed; /* inform user if limit hit */ - char kernel_verbs; /* '1' if kernel client */ + bool armed:1; /* inform user if limit hit */ + bool is_kernel_res:1; /* true if kernel client */ }; struct siw_qp_attrs { @@ -418,13 +418,11 @@ struct siw_iwarp_tx { }; struct siw_qp { + struct ib_qp base_qp; struct siw_device *sdev; - struct ib_qp *ib_qp; struct kref ref; - u32 qp_num; struct list_head devq; int tx_cpu; - bool kernel_verbs; struct siw_qp_attrs attrs; struct siw_cep *cep; @@ -472,11 +470,6 @@ struct siw_qp { struct rcu_head rcu; }; -struct siw_base_qp { - struct ib_qp base_qp; - struct siw_qp *qp; -}; - /* helper macros */ #define rx_qp(rx) container_of(rx, struct siw_qp, rx_stream) #define tx_qp(tx) container_of(tx, struct siw_qp, tx_ctx) @@ -572,14 +565,9 @@ static inline struct siw_ucontext *to_siw_ctx(struct ib_ucontext *base_ctx) return container_of(base_ctx, struct siw_ucontext, base_ucontext); } -static inline struct siw_base_qp *to_siw_base_qp(struct ib_qp *base_qp) -{ - return container_of(base_qp, struct siw_base_qp, base_qp); -} - static inline struct siw_qp *to_siw_qp(struct ib_qp *base_qp) { - return to_siw_base_qp(base_qp)->qp; + return container_of(base_qp, struct siw_qp, base_qp); } static inline struct siw_cq *to_siw_cq(struct ib_cq *base_cq) @@ -624,7 +612,7 @@ static inline struct siw_qp *siw_qp_id2obj(struct siw_device *sdev, int id) static inline u32 qp_id(struct siw_qp *qp) { - return qp->qp_num; + return qp->base_qp.qp_num; } static inline void siw_qp_get(struct siw_qp *qp) @@ -735,7 +723,7 @@ static inline void siw_crc_skb(struct siw_rx_stream *srx, unsigned int len) "MEM[0x%08x] %s: " fmt, mem->stag, __func__, ##__VA_ARGS__) #define siw_dbg_cep(cep, fmt, ...) \ - ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%pK] %s: " fmt, \ + ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%pK] %s: " fmt, \ cep, __func__, ##__VA_ARGS__) void siw_cq_flush(struct siw_cq *cq); diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c index 3bccfef40e7e..0c3f0588346e 100644 --- a/drivers/infiniband/sw/siw/siw_cm.c +++ b/drivers/infiniband/sw/siw/siw_cm.c @@ -29,7 +29,7 @@ * MPA_V2_RDMA_NO_RTR, MPA_V2_RDMA_READ_RTR, MPA_V2_RDMA_WRITE_RTR */ static __be16 rtr_type = MPA_V2_RDMA_READ_RTR | MPA_V2_RDMA_WRITE_RTR; -static const bool relaxed_ird_negotiation = 1; +static const bool relaxed_ird_negotiation = true; static void siw_cm_llp_state_change(struct sock *s); static void siw_cm_llp_data_ready(struct sock *s); diff --git a/drivers/infiniband/sw/siw/siw_cq.c b/drivers/infiniband/sw/siw/siw_cq.c index d8db3bee9da7..d68e37859e73 100644 --- a/drivers/infiniband/sw/siw/siw_cq.c +++ b/drivers/infiniband/sw/siw/siw_cq.c @@ -65,7 +65,7 @@ int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc) * reaped here, which do not hold a QP reference * and do not qualify for memory extension verbs. */ - if (likely(cq->kernel_verbs)) { + if (likely(rdma_is_kernel_res(&cq->base_cq.res))) { if (cqe->flags & SIW_WQE_REM_INVAL) { wc->ex.invalidate_rkey = cqe->inval_stag; wc->wc_flags = IB_WC_WITH_INVALIDATE; diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c index c147f0613d95..96ed349c0939 100644 --- a/drivers/infiniband/sw/siw/siw_main.c +++ b/drivers/infiniband/sw/siw/siw_main.c @@ -244,7 +244,7 @@ static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id) * siw_qp_id2obj() increments object reference count */ siw_qp_put(qp); - return qp->ib_qp; + return &qp->base_qp; } return NULL; } diff --git a/drivers/infiniband/sw/siw/siw_qp.c b/drivers/infiniband/sw/siw/siw_qp.c index b4317480cee7..875d36d4b1c6 100644 --- a/drivers/infiniband/sw/siw/siw_qp.c +++ b/drivers/infiniband/sw/siw/siw_qp.c @@ -1070,8 +1070,8 @@ int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, cqe->imm_data = 0; cqe->bytes = bytes; - if (cq->kernel_verbs) - cqe->base_qp = qp->ib_qp; + if (rdma_is_kernel_res(&cq->base_cq.res)) + cqe->base_qp = &qp->base_qp; else cqe->qp_id = qp_id(qp); @@ -1128,8 +1128,8 @@ int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, cqe->imm_data = 0; cqe->bytes = bytes; - if (cq->kernel_verbs) { - cqe->base_qp = qp->ib_qp; + if (rdma_is_kernel_res(&cq->base_cq.res)) { + cqe->base_qp = &qp->base_qp; if (inval_stag) { cqe_flags |= SIW_WQE_REM_INVAL; cqe->inval_stag = inval_stag; @@ -1297,13 +1297,12 @@ void siw_rq_flush(struct siw_qp *qp) int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) { - int rv = xa_alloc(&sdev->qp_xa, &qp->ib_qp->qp_num, qp, xa_limit_32b, + int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, GFP_KERNEL); if (!rv) { kref_init(&qp->ref); qp->sdev = sdev; - qp->qp_num = qp->ib_qp->qp_num; siw_dbg_qp(qp, "new QP\n"); } return rv; @@ -1312,7 +1311,6 @@ int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) void siw_free_qp(struct kref *ref) { struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref); - struct siw_base_qp *siw_base_qp = to_siw_base_qp(qp->ib_qp); struct siw_device *sdev = qp->sdev; unsigned long flags; @@ -1335,5 +1333,4 @@ void siw_free_qp(struct kref *ref) atomic_dec(&sdev->num_qp); siw_dbg_qp(qp, "free QP\n"); kfree_rcu(qp, rcu); - kfree(siw_base_qp); } diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index c0a887240325..9ccce2909ac4 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -68,7 +68,7 @@ static int siw_rx_umem(struct siw_rx_stream *srx, struct siw_umem *umem, return -EFAULT; } if (srx->mpa_crc_hd) { - if (rx_qp(srx)->kernel_verbs) { + if (rdma_is_kernel_res(&rx_qp(srx)->base_qp.res)) { crypto_shash_update(srx->mpa_crc_hd, (u8 *)(dest + pg_off), bytes); kunmap_atomic(dest); @@ -388,7 +388,7 @@ static struct siw_wqe *siw_rqe_get(struct siw_qp *qp) struct siw_rqe *rqe2 = &srq->recvq[off]; if (!(rqe2->flags & SIW_WQE_VALID)) { - srq->armed = 0; + srq->armed = false; srq_event = true; } } @@ -1264,7 +1264,7 @@ static int siw_rdmap_complete(struct siw_qp *qp, int error) if (wc_status == SIW_WC_SUCCESS) wc_status = SIW_WC_GENERAL_ERR; - } else if (qp->kernel_verbs && + } else if (rdma_is_kernel_res(&qp->base_qp.res) && rx_type(wqe) == SIW_OP_READ_LOCAL_INV) { /* * Handle any STag invalidation request diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c index 5d97bba0ce6d..ae92c8080967 100644 --- a/drivers/infiniband/sw/siw/siw_qp_tx.c +++ b/drivers/infiniband/sw/siw/siw_qp_tx.c @@ -817,7 +817,7 @@ static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe) } } else { wqe->bytes = wqe->sqe.sge[0].length; - if (!qp->kernel_verbs) { + if (!rdma_is_kernel_res(&qp->base_qp.res)) { if (wqe->bytes > SIW_MAX_INLINE) { rv = -EINVAL; goto tx_error; diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index 5fd6d6499b3d..07e30138aaa1 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -303,7 +303,6 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, struct ib_udata *udata) { struct siw_qp *qp = NULL; - struct siw_base_qp *siw_base_qp = NULL; struct ib_device *base_dev = pd->device; struct siw_device *sdev = to_siw_dev(base_dev); struct siw_ucontext *uctx = @@ -357,26 +356,16 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, rv = -EINVAL; goto err_out; } - siw_base_qp = kzalloc(sizeof(*siw_base_qp), GFP_KERNEL); - if (!siw_base_qp) { - rv = -ENOMEM; - goto err_out; - } qp = kzalloc(sizeof(*qp), GFP_KERNEL); if (!qp) { rv = -ENOMEM; goto err_out; } - siw_base_qp->qp = qp; - qp->ib_qp = &siw_base_qp->base_qp; - init_rwsem(&qp->state_lock); spin_lock_init(&qp->sq_lock); spin_lock_init(&qp->rq_lock); spin_lock_init(&qp->orq_lock); - qp->kernel_verbs = !udata; - rv = siw_qp_add(sdev, qp); if (rv) goto err_out; @@ -389,10 +378,10 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr); num_rqe = roundup_pow_of_two(attrs->cap.max_recv_wr); - if (qp->kernel_verbs) - qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe)); - else + if (udata) qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe)); + else + qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe)); if (qp->sendq == NULL) { siw_dbg(base_dev, "SQ size %d alloc failed\n", num_sqe); @@ -419,13 +408,14 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, */ qp->srq = to_siw_srq(attrs->srq); qp->attrs.rq_size = 0; - siw_dbg(base_dev, "QP [%u]: SRQ attached\n", qp->qp_num); + siw_dbg(base_dev, "QP [%u]: SRQ attached\n", + qp->base_qp.qp_num); } else if (num_rqe) { - if (qp->kernel_verbs) - qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe)); - else + if (udata) qp->recvq = vmalloc_user(num_rqe * sizeof(struct siw_rqe)); + else + qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe)); if (qp->recvq == NULL) { siw_dbg(base_dev, "RQ size %d alloc failed\n", num_rqe); @@ -492,13 +482,11 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, list_add_tail(&qp->devq, &sdev->qp_list); spin_unlock_irqrestore(&sdev->lock, flags); - return qp->ib_qp; + return &qp->base_qp; err_out_xa: xa_erase(&sdev->qp_xa, qp_id(qp)); err_out: - kfree(siw_base_qp); - if (qp) { if (uctx) { rdma_user_mmap_entry_remove(qp->sq_entry); @@ -742,7 +730,7 @@ int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr, unsigned long flags; int rv = 0; - if (wr && !qp->kernel_verbs) { + if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) { siw_dbg_qp(qp, "wr must be empty for user mapped sq\n"); *bad_wr = wr; return -EINVAL; @@ -939,7 +927,7 @@ int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr, if (rv <= 0) goto skip_direct_sending; - if (qp->kernel_verbs) { + if (rdma_is_kernel_res(&qp->base_qp.res)) { rv = siw_sq_start(qp); } else { qp->tx_ctx.in_syscall = 1; @@ -984,8 +972,8 @@ int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr, *bad_wr = wr; return -EOPNOTSUPP; /* what else from errno.h? */ } - if (!qp->kernel_verbs) { - siw_dbg_qp(qp, "no kernel post_recv for user mapped sq\n"); + if (!rdma_is_kernel_res(&qp->base_qp.res)) { + siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n"); *bad_wr = wr; return -EINVAL; } @@ -1127,14 +1115,13 @@ int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr, cq->base_cq.cqe = size; cq->num_cqe = size; - if (!udata) { - cq->kernel_verbs = 1; - cq->queue = vzalloc(size * sizeof(struct siw_cqe) + - sizeof(struct siw_cq_ctrl)); - } else { + if (udata) cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) + sizeof(struct siw_cq_ctrl)); - } + else + cq->queue = vzalloc(size * sizeof(struct siw_cqe) + + sizeof(struct siw_cq_ctrl)); + if (cq->queue == NULL) { rv = -ENOMEM; goto err_out; @@ -1589,9 +1576,9 @@ int siw_create_srq(struct ib_srq *base_srq, srq->num_rqe = roundup_pow_of_two(attrs->max_wr); srq->limit = attrs->srq_limit; if (srq->limit) - srq->armed = 1; + srq->armed = true; - srq->kernel_verbs = !udata; + srq->is_kernel_res = !udata; if (udata) srq->recvq = @@ -1671,9 +1658,9 @@ int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs, rv = -EINVAL; goto out; } - srq->armed = 1; + srq->armed = true; } else { - srq->armed = 0; + srq->armed = false; } srq->limit = attrs->srq_limit; } @@ -1745,7 +1732,7 @@ int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr, unsigned long flags; int rv = 0; - if (unlikely(!srq->kernel_verbs)) { + if (unlikely(!srq->is_kernel_res)) { siw_dbg_pd(base_srq->pd, "[SRQ]: no kernel post_recv for mapped srq\n"); rv = -EINVAL; @@ -1797,7 +1784,7 @@ out: void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype) { struct ib_event event; - struct ib_qp *base_qp = qp->ib_qp; + struct ib_qp *base_qp = &qp->base_qp; /* * Do not report asynchronous errors on QP which gets diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c index 0f74dc6d12fa..7a8f24de3631 100644 --- a/drivers/infiniband/ulp/iser/iser_memory.c +++ b/drivers/infiniband/ulp/iser/iser_memory.c @@ -527,7 +527,7 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task, if (unlikely(err)) goto err_reg; - desc->sig_protected = 1; + desc->sig_protected = true; } return 0; diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index 1f4a37a3c2b3..127887c6c03f 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -1093,7 +1093,7 @@ u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task, int ret; if (desc && desc->sig_protected) { - desc->sig_protected = 0; + desc->sig_protected = false; ret = ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); if (ret) { diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h index e4c9bf2ef7e2..4480092c68e0 100644 --- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h +++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_encap.h @@ -358,7 +358,7 @@ struct opa_veswport_summary_counters { * @rx_drop_state: received packets in non-forwarding port state * @rx_logic: other receive errors * - * All the above are counters of corresponding erorr conditions. + * All the above are counters of corresponding error conditions. */ struct opa_veswport_error_counters { __be16 vp_instance; diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index b7f7a5f7bd98..cd1181c39ed2 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -2546,7 +2546,8 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id, if (lrsp->opcode == SRP_LOGIN_RSP) { ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); ch->req_lim = be32_to_cpu(lrsp->req_lim_delta); - ch->use_imm_data = lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP; + ch->use_imm_data = srp_use_imm_data && + (lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP); ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, ch->use_imm_data, target->max_it_iu_size); diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 23c782e3d49a..98552749d71c 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -2810,8 +2810,6 @@ static void srpt_queue_response(struct se_cmd *cmd) int resp_len, ret, i; u8 srp_tm_status; - BUG_ON(!ch); - state = ioctx->state; switch (state) { case SRPT_STATE_NEW: diff --git a/drivers/input/input.c b/drivers/input/input.c index ee6c3234df36..fce43e62dd45 100644 --- a/drivers/input/input.c +++ b/drivers/input/input.c @@ -1216,13 +1216,12 @@ static int input_proc_devices_open(struct inode *inode, struct file *file) return seq_open(file, &input_devices_seq_ops); } -static const struct file_operations input_devices_fileops = { - .owner = THIS_MODULE, - .open = input_proc_devices_open, - .poll = input_proc_devices_poll, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release, +static const struct proc_ops input_devices_proc_ops = { + .proc_open = input_proc_devices_open, + .proc_poll = input_proc_devices_poll, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = seq_release, }; static void *input_handlers_seq_start(struct seq_file *seq, loff_t *pos) @@ -1280,12 +1279,11 @@ static int input_proc_handlers_open(struct inode *inode, struct file *file) return seq_open(file, &input_handlers_seq_ops); } -static const struct file_operations input_handlers_fileops = { - .owner = THIS_MODULE, - .open = input_proc_handlers_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release, +static const struct proc_ops input_handlers_proc_ops = { + .proc_open = input_proc_handlers_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = seq_release, }; static int __init input_proc_init(void) @@ -1297,12 +1295,12 @@ static int __init input_proc_init(void) return -ENOMEM; entry = proc_create("devices", 0, proc_bus_input_dir, - &input_devices_fileops); + &input_devices_proc_ops); if (!entry) goto fail1; entry = proc_create("handlers", 0, proc_bus_input_dir, - &input_handlers_fileops); + &input_handlers_proc_ops); if (!entry) goto fail2; diff --git a/drivers/input/misc/axp20x-pek.c b/drivers/input/misc/axp20x-pek.c index 17c1cca74498..c8f87df93a50 100644 --- a/drivers/input/misc/axp20x-pek.c +++ b/drivers/input/misc/axp20x-pek.c @@ -191,9 +191,10 @@ static ssize_t axp20x_store_attr_shutdown(struct device *dev, axp20x_pek->info->shutdown_mask, buf, count); } -DEVICE_ATTR(startup, 0644, axp20x_show_attr_startup, axp20x_store_attr_startup); -DEVICE_ATTR(shutdown, 0644, axp20x_show_attr_shutdown, - axp20x_store_attr_shutdown); +static DEVICE_ATTR(startup, 0644, axp20x_show_attr_startup, + axp20x_store_attr_startup); +static DEVICE_ATTR(shutdown, 0644, axp20x_show_attr_shutdown, + axp20x_store_attr_shutdown); static struct attribute *axp20x_attrs[] = { &dev_attr_startup.attr, @@ -279,8 +280,7 @@ static int axp20x_pek_probe_input_device(struct axp20x_pek *axp20x_pek, return error; } - if (axp20x_pek->axp20x->variant == AXP288_ID) - enable_irq_wake(axp20x_pek->irq_dbr); + device_init_wakeup(&pdev->dev, true); return 0; } @@ -352,6 +352,40 @@ static int axp20x_pek_probe(struct platform_device *pdev) return 0; } +static int __maybe_unused axp20x_pek_suspend(struct device *dev) +{ + struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev); + + /* + * As nested threaded IRQs are not automatically disabled during + * suspend, we must explicitly disable non-wakeup IRQs. + */ + if (device_may_wakeup(dev)) { + enable_irq_wake(axp20x_pek->irq_dbf); + enable_irq_wake(axp20x_pek->irq_dbr); + } else { + disable_irq(axp20x_pek->irq_dbf); + disable_irq(axp20x_pek->irq_dbr); + } + + return 0; +} + +static int __maybe_unused axp20x_pek_resume(struct device *dev) +{ + struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev); + + if (device_may_wakeup(dev)) { + disable_irq_wake(axp20x_pek->irq_dbf); + disable_irq_wake(axp20x_pek->irq_dbr); + } else { + enable_irq(axp20x_pek->irq_dbf); + enable_irq(axp20x_pek->irq_dbr); + } + + return 0; +} + static int __maybe_unused axp20x_pek_resume_noirq(struct device *dev) { struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev); @@ -371,6 +405,7 @@ static int __maybe_unused axp20x_pek_resume_noirq(struct device *dev) } static const struct dev_pm_ops axp20x_pek_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(axp20x_pek_suspend, axp20x_pek_resume) #ifdef CONFIG_PM_SLEEP .resume_noirq = axp20x_pek_resume_noirq, #endif diff --git a/drivers/input/rmi4/rmi_f11.c b/drivers/input/rmi4/rmi_f11.c index bbf9ae9f3f0c..6adea8a3e8fb 100644 --- a/drivers/input/rmi4/rmi_f11.c +++ b/drivers/input/rmi4/rmi_f11.c @@ -412,6 +412,10 @@ struct f11_2d_sensor_queries { /* Defs for Ctrl0. */ #define RMI_F11_REPORT_MODE_MASK 0x07 +#define RMI_F11_REPORT_MODE_CONTINUOUS (0 << 0) +#define RMI_F11_REPORT_MODE_REDUCED (1 << 0) +#define RMI_F11_REPORT_MODE_FS_CHANGE (2 << 0) +#define RMI_F11_REPORT_MODE_FP_CHANGE (3 << 0) #define RMI_F11_ABS_POS_FILT (1 << 3) #define RMI_F11_REL_POS_FILT (1 << 4) #define RMI_F11_REL_BALLISTICS (1 << 5) @@ -1195,6 +1199,16 @@ static int rmi_f11_initialize(struct rmi_function *fn) ctrl->ctrl0_11[RMI_F11_DELTA_Y_THRESHOLD] = sensor->axis_align.delta_y_threshold; + /* + * If distance threshold values are set, switch to reduced reporting + * mode so they actually get used by the controller. + */ + if (ctrl->ctrl0_11[RMI_F11_DELTA_X_THRESHOLD] || + ctrl->ctrl0_11[RMI_F11_DELTA_Y_THRESHOLD]) { + ctrl->ctrl0_11[0] &= ~RMI_F11_REPORT_MODE_MASK; + ctrl->ctrl0_11[0] |= RMI_F11_REPORT_MODE_REDUCED; + } + if (f11->sens_query.has_dribble) { switch (sensor->dribble) { case RMI_REG_STATE_OFF: diff --git a/drivers/input/serio/Kconfig b/drivers/input/serio/Kconfig index f3e18f8ef9ca..373a1646019e 100644 --- a/drivers/input/serio/Kconfig +++ b/drivers/input/serio/Kconfig @@ -165,6 +165,16 @@ config SERIO_MACEPS2 To compile this driver as a module, choose M here: the module will be called maceps2. +config SERIO_SGI_IOC3 + tristate "SGI IOC3 PS/2 controller" + depends on SGI_MFD_IOC3 + help + Say Y here if you have an SGI Onyx2, SGI Octane or IOC3 PCI card + and you want to attach and use a keyboard, mouse, or both. + + To compile this driver as a module, choose M here: the + module will be called ioc3kbd. + config SERIO_LIBPS2 tristate "PS/2 driver library" depends on SERIO_I8042 || SERIO_I8042=n diff --git a/drivers/input/serio/Makefile b/drivers/input/serio/Makefile index 67950a5ccb3f..6d97bad7b844 100644 --- a/drivers/input/serio/Makefile +++ b/drivers/input/serio/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_HIL_MLC) += hp_sdc_mlc.o hil_mlc.o obj-$(CONFIG_SERIO_PCIPS2) += pcips2.o obj-$(CONFIG_SERIO_PS2MULT) += ps2mult.o obj-$(CONFIG_SERIO_MACEPS2) += maceps2.o +obj-$(CONFIG_SERIO_SGI_IOC3) += ioc3kbd.o obj-$(CONFIG_SERIO_LIBPS2) += libps2.o obj-$(CONFIG_SERIO_RAW) += serio_raw.o obj-$(CONFIG_SERIO_AMS_DELTA) += ams_delta_serio.o diff --git a/drivers/input/serio/apbps2.c b/drivers/input/serio/apbps2.c index f290d5d146c3..594ac4e6f8ea 100644 --- a/drivers/input/serio/apbps2.c +++ b/drivers/input/serio/apbps2.c @@ -51,7 +51,7 @@ struct apbps2_regs { struct apbps2_priv { struct serio *io; - struct apbps2_regs *regs; + struct apbps2_regs __iomem *regs; }; static int apbps2_idx; diff --git a/drivers/input/serio/hyperv-keyboard.c b/drivers/input/serio/hyperv-keyboard.c index e486a8a74c40..df4e9f6f4529 100644 --- a/drivers/input/serio/hyperv-keyboard.c +++ b/drivers/input/serio/hyperv-keyboard.c @@ -259,6 +259,8 @@ static int hv_kbd_connect_to_vsp(struct hv_device *hv_dev) u32 proto_status; int error; + reinit_completion(&kbd_dev->wait_event); + request = &kbd_dev->protocol_req; memset(request, 0, sizeof(struct synth_kbd_protocol_request)); request->header.type = __cpu_to_le32(SYNTH_KBD_PROTOCOL_REQUEST); @@ -380,6 +382,29 @@ static int hv_kbd_remove(struct hv_device *hv_dev) return 0; } +static int hv_kbd_suspend(struct hv_device *hv_dev) +{ + vmbus_close(hv_dev->channel); + + return 0; +} + +static int hv_kbd_resume(struct hv_device *hv_dev) +{ + int ret; + + ret = vmbus_open(hv_dev->channel, + KBD_VSC_SEND_RING_BUFFER_SIZE, + KBD_VSC_RECV_RING_BUFFER_SIZE, + NULL, 0, + hv_kbd_on_channel_callback, + hv_dev); + if (ret == 0) + ret = hv_kbd_connect_to_vsp(hv_dev); + + return ret; +} + static const struct hv_vmbus_device_id id_table[] = { /* Keyboard guid */ { HV_KBD_GUID, }, @@ -393,6 +418,8 @@ static struct hv_driver hv_kbd_drv = { .id_table = id_table, .probe = hv_kbd_probe, .remove = hv_kbd_remove, + .suspend = hv_kbd_suspend, + .resume = hv_kbd_resume, .driver = { .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, diff --git a/drivers/input/serio/ioc3kbd.c b/drivers/input/serio/ioc3kbd.c new file mode 100644 index 000000000000..d51bfe912db5 --- /dev/null +++ b/drivers/input/serio/ioc3kbd.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SGI IOC3 PS/2 controller driver for linux + * + * Copyright (C) 2019 Thomas Bogendoerfer <tbogendoerfer@suse.de> + * + * Based on code Copyright (C) 2005 Stanislaw Skowronek <skylark@unaligned.org> + * Copyright (C) 2009 Johannes Dickgreber <tanzy@gmx.de> + */ + +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/serio.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +#include <asm/sn/ioc3.h> + +struct ioc3kbd_data { + struct ioc3_serioregs __iomem *regs; + struct serio *kbd, *aux; + bool kbd_exists, aux_exists; + int irq; +}; + +static int ioc3kbd_wait(struct ioc3_serioregs __iomem *regs, u32 mask) +{ + unsigned long timeout = 0; + + while ((readl(®s->km_csr) & mask) && (timeout < 250)) { + udelay(50); + timeout++; + } + return (timeout >= 250) ? -ETIMEDOUT : 0; +} + +static int ioc3kbd_write(struct serio *dev, u8 val) +{ + struct ioc3kbd_data *d = dev->port_data; + int ret; + + ret = ioc3kbd_wait(d->regs, KM_CSR_K_WRT_PEND); + if (ret) + return ret; + + writel(val, &d->regs->k_wd); + + return 0; +} + +static int ioc3kbd_start(struct serio *dev) +{ + struct ioc3kbd_data *d = dev->port_data; + + d->kbd_exists = true; + return 0; +} + +static void ioc3kbd_stop(struct serio *dev) +{ + struct ioc3kbd_data *d = dev->port_data; + + d->kbd_exists = false; +} + +static int ioc3aux_write(struct serio *dev, u8 val) +{ + struct ioc3kbd_data *d = dev->port_data; + int ret; + + ret = ioc3kbd_wait(d->regs, KM_CSR_M_WRT_PEND); + if (ret) + return ret; + + writel(val, &d->regs->m_wd); + + return 0; +} + +static int ioc3aux_start(struct serio *dev) +{ + struct ioc3kbd_data *d = dev->port_data; + + d->aux_exists = true; + return 0; +} + +static void ioc3aux_stop(struct serio *dev) +{ + struct ioc3kbd_data *d = dev->port_data; + + d->aux_exists = false; +} + +static void ioc3kbd_process_data(struct serio *dev, u32 data) +{ + if (data & KM_RD_VALID_0) + serio_interrupt(dev, (data >> KM_RD_DATA_0_SHIFT) & 0xff, 0); + if (data & KM_RD_VALID_1) + serio_interrupt(dev, (data >> KM_RD_DATA_1_SHIFT) & 0xff, 0); + if (data & KM_RD_VALID_2) + serio_interrupt(dev, (data >> KM_RD_DATA_2_SHIFT) & 0xff, 0); +} + +static irqreturn_t ioc3kbd_intr(int itq, void *dev_id) +{ + struct ioc3kbd_data *d = dev_id; + u32 data_k, data_m; + + data_k = readl(&d->regs->k_rd); + if (d->kbd_exists) + ioc3kbd_process_data(d->kbd, data_k); + + data_m = readl(&d->regs->m_rd); + if (d->aux_exists) + ioc3kbd_process_data(d->aux, data_m); + + return IRQ_HANDLED; +} + +static int ioc3kbd_probe(struct platform_device *pdev) +{ + struct ioc3_serioregs __iomem *regs; + struct device *dev = &pdev->dev; + struct ioc3kbd_data *d; + struct serio *sk, *sa; + int irq, ret; + + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return -ENXIO; + + d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + sk = kzalloc(sizeof(*sk), GFP_KERNEL); + if (!sk) + return -ENOMEM; + + sa = kzalloc(sizeof(*sa), GFP_KERNEL); + if (!sa) { + kfree(sk); + return -ENOMEM; + } + + sk->id.type = SERIO_8042; + sk->write = ioc3kbd_write; + sk->start = ioc3kbd_start; + sk->stop = ioc3kbd_stop; + snprintf(sk->name, sizeof(sk->name), "IOC3 keyboard %d", pdev->id); + snprintf(sk->phys, sizeof(sk->phys), "ioc3/serio%dkbd", pdev->id); + sk->port_data = d; + sk->dev.parent = dev; + + sa->id.type = SERIO_8042; + sa->write = ioc3aux_write; + sa->start = ioc3aux_start; + sa->stop = ioc3aux_stop; + snprintf(sa->name, sizeof(sa->name), "IOC3 auxiliary %d", pdev->id); + snprintf(sa->phys, sizeof(sa->phys), "ioc3/serio%daux", pdev->id); + sa->port_data = d; + sa->dev.parent = dev; + + d->regs = regs; + d->kbd = sk; + d->aux = sa; + d->irq = irq; + + platform_set_drvdata(pdev, d); + serio_register_port(d->kbd); + serio_register_port(d->aux); + + ret = request_irq(irq, ioc3kbd_intr, IRQF_SHARED, "ioc3-kbd", d); + if (ret) { + dev_err(dev, "could not request IRQ %d\n", irq); + serio_unregister_port(d->kbd); + serio_unregister_port(d->aux); + return ret; + } + + /* enable ports */ + writel(KM_CSR_K_CLAMP_3 | KM_CSR_M_CLAMP_3, ®s->km_csr); + + return 0; +} + +static int ioc3kbd_remove(struct platform_device *pdev) +{ + struct ioc3kbd_data *d = platform_get_drvdata(pdev); + + free_irq(d->irq, d); + + serio_unregister_port(d->kbd); + serio_unregister_port(d->aux); + + return 0; +} + +static struct platform_driver ioc3kbd_driver = { + .probe = ioc3kbd_probe, + .remove = ioc3kbd_remove, + .driver = { + .name = "ioc3-kbd", + }, +}; +module_platform_driver(ioc3kbd_driver); + +MODULE_AUTHOR("Thomas Bogendoerfer <tbogendoerfer@suse.de>"); +MODULE_DESCRIPTION("SGI IOC3 serio driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/input/touchscreen/ads7846.c b/drivers/input/touchscreen/ads7846.c index 51ddb204ca1b..8fd7fc39c4fd 100644 --- a/drivers/input/touchscreen/ads7846.c +++ b/drivers/input/touchscreen/ads7846.c @@ -333,7 +333,8 @@ static int ads7846_read12_ser(struct device *dev, unsigned command) req->xfer[1].len = 2; /* for 1uF, settle for 800 usec; no cap, 100 usec. */ - req->xfer[1].delay_usecs = ts->vref_delay_usecs; + req->xfer[1].delay.value = ts->vref_delay_usecs; + req->xfer[1].delay.unit = SPI_DELAY_UNIT_USECS; spi_message_add_tail(&req->xfer[1], &req->msg); /* Enable reference voltage */ @@ -1018,7 +1019,8 @@ static void ads7846_setup_spi_msg(struct ads7846 *ts, * have had enough time to stabilize. */ if (pdata->settle_delay_usecs) { - x->delay_usecs = pdata->settle_delay_usecs; + x->delay.value = pdata->settle_delay_usecs; + x->delay.unit = SPI_DELAY_UNIT_USECS; x++; x->tx_buf = &packet->read_y; @@ -1061,7 +1063,8 @@ static void ads7846_setup_spi_msg(struct ads7846 *ts, /* ... maybe discard first sample ... */ if (pdata->settle_delay_usecs) { - x->delay_usecs = pdata->settle_delay_usecs; + x->delay.value = pdata->settle_delay_usecs; + x->delay.unit = SPI_DELAY_UNIT_USECS; x++; x->tx_buf = &packet->read_x; @@ -1094,7 +1097,8 @@ static void ads7846_setup_spi_msg(struct ads7846 *ts, /* ... maybe discard first sample ... */ if (pdata->settle_delay_usecs) { - x->delay_usecs = pdata->settle_delay_usecs; + x->delay.value = pdata->settle_delay_usecs; + x->delay.unit = SPI_DELAY_UNIT_USECS; x++; x->tx_buf = &packet->read_z1; @@ -1125,7 +1129,8 @@ static void ads7846_setup_spi_msg(struct ads7846 *ts, /* ... maybe discard first sample ... */ if (pdata->settle_delay_usecs) { - x->delay_usecs = pdata->settle_delay_usecs; + x->delay.value = pdata->settle_delay_usecs; + x->delay.unit = SPI_DELAY_UNIT_USECS; x++; x->tx_buf = &packet->read_z2; diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c index d61731c0037d..d2587724c52a 100644 --- a/drivers/input/touchscreen/edt-ft5x06.c +++ b/drivers/input/touchscreen/edt-ft5x06.c @@ -13,22 +13,23 @@ * http://www.glyn.com/Products/Displays */ -#include <linux/module.h> -#include <linux/ratelimit.h> -#include <linux/irq.h> -#include <linux/interrupt.h> -#include <linux/input.h> -#include <linux/i2c.h> -#include <linux/kernel.h> -#include <linux/uaccess.h> -#include <linux/delay.h> #include <linux/debugfs.h> -#include <linux/slab.h> +#include <linux/delay.h> #include <linux/gpio/consumer.h> +#include <linux/i2c.h> +#include <linux/interrupt.h> +#include <linux/input.h> #include <linux/input/mt.h> #include <linux/input/touchscreen.h> -#include <asm/unaligned.h> +#include <linux/irq.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/ratelimit.h> #include <linux/regulator/consumer.h> +#include <linux/slab.h> +#include <linux/uaccess.h> + +#include <asm/unaligned.h> #define WORK_REGISTER_THRESHOLD 0x00 #define WORK_REGISTER_REPORT_RATE 0x08 @@ -1050,6 +1051,7 @@ static int edt_ft5x06_ts_probe(struct i2c_client *client, { const struct edt_i2c_chip_data *chip_data; struct edt_ft5x06_ts_data *tsdata; + u8 buf[2] = { 0xfc, 0x00 }; struct input_dev *input; unsigned long irq_flags; int error; @@ -1140,6 +1142,12 @@ static int edt_ft5x06_ts_probe(struct i2c_client *client, return error; } + /* + * Dummy read access. EP0700MLP1 returns bogus data on the first + * register read access and ignores writes. + */ + edt_ft5x06_ts_readwrite(tsdata->client, 2, buf, 2, buf); + edt_ft5x06_ts_set_regs(tsdata); edt_ft5x06_ts_get_defaults(&client->dev, tsdata); edt_ft5x06_ts_get_parameters(tsdata); @@ -1200,7 +1208,6 @@ static int edt_ft5x06_ts_probe(struct i2c_client *client, return error; edt_ft5x06_ts_prepare_debugfs(tsdata, dev_driver_string(&client->dev)); - device_init_wakeup(&client->dev, 1); dev_dbg(&client->dev, "EDT FT5x06 initialized: IRQ %d, WAKE pin %d, Reset pin %d.\n", @@ -1220,29 +1227,6 @@ static int edt_ft5x06_ts_remove(struct i2c_client *client) return 0; } -static int __maybe_unused edt_ft5x06_ts_suspend(struct device *dev) -{ - struct i2c_client *client = to_i2c_client(dev); - - if (device_may_wakeup(dev)) - enable_irq_wake(client->irq); - - return 0; -} - -static int __maybe_unused edt_ft5x06_ts_resume(struct device *dev) -{ - struct i2c_client *client = to_i2c_client(dev); - - if (device_may_wakeup(dev)) - disable_irq_wake(client->irq); - - return 0; -} - -static SIMPLE_DEV_PM_OPS(edt_ft5x06_ts_pm_ops, - edt_ft5x06_ts_suspend, edt_ft5x06_ts_resume); - static const struct edt_i2c_chip_data edt_ft5x06_data = { .max_support_points = 5, }; @@ -1281,7 +1265,6 @@ static struct i2c_driver edt_ft5x06_ts_driver = { .driver = { .name = "edt_ft5x06", .of_match_table = edt_ft5x06_of_match, - .pm = &edt_ft5x06_ts_pm_ops, }, .id_table = edt_ft5x06_ts_id, .probe = edt_ft5x06_ts_probe, diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c index d4ad24ea54c8..491179967b29 100644 --- a/drivers/input/touchscreen/elants_i2c.c +++ b/drivers/input/touchscreen/elants_i2c.c @@ -59,8 +59,10 @@ #define CMD_HEADER_WRITE 0x54 #define CMD_HEADER_READ 0x53 #define CMD_HEADER_6B_READ 0x5B +#define CMD_HEADER_ROM_READ 0x96 #define CMD_HEADER_RESP 0x52 #define CMD_HEADER_6B_RESP 0x9B +#define CMD_HEADER_ROM_RESP 0x95 #define CMD_HEADER_HELLO 0x55 #define CMD_HEADER_REK 0x66 @@ -200,6 +202,10 @@ static int elants_i2c_execute_command(struct i2c_client *client, expected_response = CMD_HEADER_6B_RESP; break; + case CMD_HEADER_ROM_READ: + expected_response = CMD_HEADER_ROM_RESP; + break; + default: dev_err(&client->dev, "%s: invalid command %*ph\n", __func__, (int)cmd_size, cmd); @@ -556,6 +562,8 @@ static int elants_i2c_initialize(struct elants_data *ts) /* hw version is available even if device in recovery state */ error2 = elants_i2c_query_hw_version(ts); + if (!error2) + error2 = elants_i2c_query_bc_version(ts); if (!error) error = error2; @@ -564,8 +572,6 @@ static int elants_i2c_initialize(struct elants_data *ts) if (!error) error = elants_i2c_query_test_version(ts); if (!error) - error = elants_i2c_query_bc_version(ts); - if (!error) error = elants_i2c_query_ts_info(ts); if (error) @@ -613,39 +619,94 @@ static int elants_i2c_fw_write_page(struct i2c_client *client, return error; } +static int elants_i2c_validate_remark_id(struct elants_data *ts, + const struct firmware *fw) +{ + struct i2c_client *client = ts->client; + int error; + const u8 cmd[] = { CMD_HEADER_ROM_READ, 0x80, 0x1F, 0x00, 0x00, 0x21 }; + u8 resp[6] = { 0 }; + u16 ts_remark_id = 0; + u16 fw_remark_id = 0; + + /* Compare TS Remark ID and FW Remark ID */ + error = elants_i2c_execute_command(client, cmd, sizeof(cmd), + resp, sizeof(resp)); + if (error) { + dev_err(&client->dev, "failed to query Remark ID: %d\n", error); + return error; + } + + ts_remark_id = get_unaligned_be16(&resp[3]); + + fw_remark_id = get_unaligned_le16(&fw->data[fw->size - 4]); + + if (fw_remark_id != ts_remark_id) { + dev_err(&client->dev, + "Remark ID Mismatched: ts_remark_id=0x%04x, fw_remark_id=0x%04x.\n", + ts_remark_id, fw_remark_id); + return -EINVAL; + } + + return 0; +} + static int elants_i2c_do_update_firmware(struct i2c_client *client, const struct firmware *fw, bool force) { + struct elants_data *ts = i2c_get_clientdata(client); const u8 enter_iap[] = { 0x45, 0x49, 0x41, 0x50 }; const u8 enter_iap2[] = { 0x54, 0x00, 0x12, 0x34 }; const u8 iap_ack[] = { 0x55, 0xaa, 0x33, 0xcc }; - const u8 close_idle[] = {0x54, 0x2c, 0x01, 0x01}; + const u8 close_idle[] = { 0x54, 0x2c, 0x01, 0x01 }; u8 buf[HEADER_SIZE]; u16 send_id; int page, n_fw_pages; int error; + bool check_remark_id = ts->iap_version >= 0x60; /* Recovery mode detection! */ if (force) { dev_dbg(&client->dev, "Recovery mode procedure\n"); + + if (check_remark_id) { + error = elants_i2c_validate_remark_id(ts, fw); + if (error) + return error; + } + error = elants_i2c_send(client, enter_iap2, sizeof(enter_iap2)); + if (error) { + dev_err(&client->dev, "failed to enter IAP mode: %d\n", + error); + return error; + } } else { /* Start IAP Procedure */ dev_dbg(&client->dev, "Normal IAP procedure\n"); + /* Close idle mode */ error = elants_i2c_send(client, close_idle, sizeof(close_idle)); if (error) dev_err(&client->dev, "Failed close idle: %d\n", error); msleep(60); + elants_i2c_sw_reset(client); msleep(20); - error = elants_i2c_send(client, enter_iap, sizeof(enter_iap)); - } - if (error) { - dev_err(&client->dev, "failed to enter IAP mode: %d\n", error); - return error; + if (check_remark_id) { + error = elants_i2c_validate_remark_id(ts, fw); + if (error) + return error; + } + + error = elants_i2c_send(client, enter_iap, sizeof(enter_iap)); + if (error) { + dev_err(&client->dev, "failed to enter IAP mode: %d\n", + error); + return error; + } } msleep(20); diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 0b9d78a0f3ac..d2fade984999 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -82,7 +82,7 @@ config IOMMU_DEBUGFS config IOMMU_DEFAULT_PASSTHROUGH bool "IOMMU passthrough by default" depends on IOMMU_API - help + help Enable passthrough by default, removing the need to pass in iommu.passthrough=on or iommu=pt through command line. If this is enabled, you can still disable with iommu.passthrough=off @@ -91,8 +91,8 @@ config IOMMU_DEFAULT_PASSTHROUGH If unsure, say N here. config OF_IOMMU - def_bool y - depends on OF && IOMMU_API + def_bool y + depends on OF && IOMMU_API # IOMMU-agnostic DMA-mapping layer config IOMMU_DMA @@ -214,6 +214,7 @@ config INTEL_IOMMU_SVM select PCI_PASID select PCI_PRI select MMU_NOTIFIER + select IOASID help Shared Virtual Memory (SVM) provides a facility for devices to access DMA resources through process address space by @@ -248,6 +249,18 @@ config INTEL_IOMMU_FLOPPY_WA workaround will setup a 1:1 mapping for the first 16MiB to make floppy (an ISA device) work. +config INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON + bool "Enable Intel IOMMU scalable mode by default" + depends on INTEL_IOMMU + help + Selecting this option will enable by default the scalable mode if + hardware presents the capability. The scalable mode is defined in + VT-d 3.0. The scalable mode capability could be checked by reading + /sys/devices/virtual/iommu/dmar*/intel-iommu/ecap. If this option + is not selected, scalable mode support could also be enabled by + passing intel_iommu=sm_on to the kernel. If not sure, please use + the default value. + config IRQ_REMAP bool "Support for Interrupt Remapping" depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI @@ -356,7 +369,7 @@ config SPAPR_TCE_IOMMU # ARM IOMMU support config ARM_SMMU - bool "ARM Ltd. System MMU (SMMU) Support" + tristate "ARM Ltd. System MMU (SMMU) Support" depends on (ARM64 || ARM) && MMU select IOMMU_API select IOMMU_IO_PGTABLE_LPAE @@ -368,6 +381,18 @@ config ARM_SMMU Say Y here if your SoC includes an IOMMU device implementing the ARM SMMU architecture. +config ARM_SMMU_LEGACY_DT_BINDINGS + bool "Support the legacy \"mmu-masters\" devicetree bindings" + depends on ARM_SMMU=y && OF + help + Support for the badly designed and deprecated "mmu-masters" + devicetree bindings. This allows some DMA masters to attach + to the SMMU but does not provide any support via the DMA API. + If you're lucky, you might be able to get VFIO up and running. + + If you say Y here then you'll make me very sad. Instead, say N + and move your firmware to the utopian future that was 2016. + config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT bool "Default to disabling bypass on ARM SMMU v1 and v2" depends on ARM_SMMU @@ -394,7 +419,7 @@ config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT config. config ARM_SMMU_V3 - bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support" + tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support" depends on ARM64 select IOMMU_API select IOMMU_IO_PGTABLE_LPAE diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 97814cc861ea..2104fb8afc06 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -14,7 +14,8 @@ obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o amd_iommu_quirks.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o -obj-$(CONFIG_ARM_SMMU) += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o +obj-$(CONFIG_ARM_SMMU) += arm-smmu-mod.o +arm-smmu-mod-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index bd25674ee4db..aac132bd1ef0 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -230,11 +230,8 @@ static struct pci_dev *setup_aliases(struct device *dev) */ ivrs_alias = amd_iommu_alias_table[pci_dev_id(pdev)]; if (ivrs_alias != pci_dev_id(pdev) && - PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) { - pci_add_dma_alias(pdev, ivrs_alias & 0xff); - pci_info(pdev, "Added PCI DMA alias %02x.%d\n", - PCI_SLOT(ivrs_alias), PCI_FUNC(ivrs_alias)); - } + PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) + pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1); clone_aliases(pdev); @@ -2297,7 +2294,6 @@ int __init amd_iommu_init_api(void) int __init amd_iommu_init_dma_ops(void) { swiotlb = (iommu_default_passthrough() || sme_me_mask) ? 1 : 0; - iommu_detected = 1; if (amd_iommu_unmap_flush) pr_info("IO/TLB flush on unmap enabled\n"); @@ -2641,15 +2637,6 @@ static void amd_iommu_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); } -static void amd_iommu_put_resv_regions(struct device *dev, - struct list_head *head) -{ - struct iommu_resv_region *entry, *next; - - list_for_each_entry_safe(entry, next, head, list) - kfree(entry); -} - static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain, struct device *dev) { @@ -2688,7 +2675,7 @@ const struct iommu_ops amd_iommu_ops = { .device_group = amd_iommu_device_group, .domain_get_attr = amd_iommu_domain_get_attr, .get_resv_regions = amd_iommu_get_resv_regions, - .put_resv_regions = amd_iommu_put_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, .pgsize_bitmap = AMD_IOMMU_PGSIZES, .flush_iotlb_all = amd_iommu_flush_iotlb_all, diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 823cc4ef51fd..2759a8d57b7f 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -71,6 +71,8 @@ #define IVHD_FLAG_ISOC_EN_MASK 0x08 #define IVMD_FLAG_EXCL_RANGE 0x08 +#define IVMD_FLAG_IW 0x04 +#define IVMD_FLAG_IR 0x02 #define IVMD_FLAG_UNITY_MAP 0x01 #define ACPI_DEVFLAG_INITPASS 0x01 @@ -147,7 +149,7 @@ bool amd_iommu_dump; bool amd_iommu_irq_remap __read_mostly; int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; -static int amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; +static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; static bool amd_iommu_detected; static bool __initdata amd_iommu_disabled; @@ -714,7 +716,7 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu) writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); - iommu_feature_enable(iommu, CONTROL_PPFLOG_EN); + iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); iommu_feature_enable(iommu, CONTROL_PPR_EN); } @@ -1116,21 +1118,17 @@ static int __init add_early_maps(void) */ static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m) { - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; - if (!(m->flags & IVMD_FLAG_EXCL_RANGE)) return; - if (iommu) { - /* - * We only can configure exclusion ranges per IOMMU, not - * per device. But we can enable the exclusion range per - * device. This is done here - */ - set_dev_entry_bit(devid, DEV_ENTRY_EX); - iommu->exclusion_start = m->range_start; - iommu->exclusion_length = m->range_length; - } + /* + * Treat per-device exclusion ranges as r/w unity-mapped regions + * since some buggy BIOSes might lead to the overwritten exclusion + * range (exclusion_start and exclusion_length members). This + * happens when there are multiple exclusion ranges (IVMD entries) + * defined in ACPI table. + */ + m->flags = (IVMD_FLAG_IW | IVMD_FLAG_IR | IVMD_FLAG_UNITY_MAP); } /* @@ -1523,8 +1521,6 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; - if (((h->efr_attr & (0x1 << IOMMU_FEAT_XTSUP_SHIFT)) == 0)) - amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; break; case 0x11: case 0x40: @@ -1534,8 +1530,15 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; - if (((h->efr_reg & (0x1 << IOMMU_EFR_XTSUP_SHIFT)) == 0)) - amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; + /* + * Note: Since iommu_update_intcapxt() leverages + * the IOMMU MMIO access to MSI capability block registers + * for MSI address lo/hi/data, we need to check both + * EFR[XtSup] and EFR[MsiCapMmioSup] for x2APIC support. + */ + if ((h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) && + (h->efr_reg & BIT(IOMMU_EFR_MSICAPMMIOSUP_SHIFT))) + amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; break; default: return -EINVAL; @@ -1727,7 +1730,6 @@ static const struct attribute_group *amd_iommu_groups[] = { static int __init iommu_init_pci(struct amd_iommu *iommu) { int cap_ptr = iommu->cap_ptr; - u32 range, misc, low, high; int ret; iommu->dev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(iommu->devid), @@ -1740,19 +1742,12 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, &iommu->cap); - pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET, - &range); - pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET, - &misc); if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) amd_iommu_iotlb_sup = false; /* read extended feature bits */ - low = readl(iommu->mmio_base + MMIO_EXT_FEATURES); - high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4); - - iommu->features = ((u64)high << 32) | low; + iommu->features = readq(iommu->mmio_base + MMIO_EXT_FEATURES); if (iommu_feature(iommu, FEATURE_GT)) { int glxval; @@ -1996,8 +1991,8 @@ static int iommu_init_intcapxt(struct amd_iommu *iommu) struct irq_affinity_notify *notify = &iommu->intcapxt_notify; /** - * IntCapXT requires XTSup=1, which can be inferred - * amd_iommu_xt_mode. + * IntCapXT requires XTSup=1 and MsiCapMmioSup=1, + * which can be inferred from amd_iommu_xt_mode. */ if (amd_iommu_xt_mode != IRQ_REMAP_X2APIC_MODE) return 0; @@ -2044,7 +2039,7 @@ enable_faults: iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); if (iommu->ppr_log != NULL) - iommu_feature_enable(iommu, CONTROL_PPFINT_EN); + iommu_feature_enable(iommu, CONTROL_PPRINT_EN); iommu_ga_log_enable(iommu); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index f52f59d5c6bd..f8d01d6b00da 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -147,8 +147,8 @@ #define CONTROL_COHERENT_EN 0x0aULL #define CONTROL_ISOC_EN 0x0bULL #define CONTROL_CMDBUF_EN 0x0cULL -#define CONTROL_PPFLOG_EN 0x0dULL -#define CONTROL_PPFINT_EN 0x0eULL +#define CONTROL_PPRLOG_EN 0x0dULL +#define CONTROL_PPRINT_EN 0x0eULL #define CONTROL_PPR_EN 0x0fULL #define CONTROL_GT_EN 0x10ULL #define CONTROL_GA_EN 0x11ULL @@ -377,12 +377,12 @@ #define IOMMU_CAP_EFR 27 /* IOMMU Feature Reporting Field (for IVHD type 10h */ -#define IOMMU_FEAT_XTSUP_SHIFT 0 #define IOMMU_FEAT_GASUP_SHIFT 6 /* IOMMU Extended Feature Register (EFR) */ #define IOMMU_EFR_XTSUP_SHIFT 2 #define IOMMU_EFR_GASUP_SHIFT 7 +#define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46 #define MAX_DOMAIN_ID 65536 @@ -463,7 +463,6 @@ struct amd_irte_ops; * independent of their use. */ struct protection_domain { - struct list_head list; /* for list of all protection domains */ struct list_head dev_list; /* List of all devices in this domain */ struct iommu_domain domain; /* generic domain handle used by iommu core code */ diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index b2fe72a8f019..74d97a886e93 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -119,7 +119,7 @@ int arm_mmu500_reset(struct arm_smmu_device *smmu) * Secure has also cleared SACR.CACHE_LOCK for this to take effect... */ reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7); - major = FIELD_GET(ID7_MAJOR, reg); + major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg); reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); if (major >= 2) reg &= ~ARM_MMU500_ACR_CACHE_LOCK; diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index effe72eb89e7..aa3ac2a03807 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -21,8 +21,7 @@ #include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/iopoll.h> -#include <linux/init.h> -#include <linux/moduleparam.h> +#include <linux/module.h> #include <linux/msi.h> #include <linux/of.h> #include <linux/of_address.h> @@ -224,9 +223,15 @@ #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4) #define STRTAB_STE_0_S1FMT_LINEAR 0 +#define STRTAB_STE_0_S1FMT_64K_L2 2 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59) +#define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0) +#define STRTAB_STE_1_S1DSS_TERMINATE 0x0 +#define STRTAB_STE_1_S1DSS_BYPASS 0x1 +#define STRTAB_STE_1_S1DSS_SSID0 0x2 + #define STRTAB_STE_1_S1C_CACHE_NC 0UL #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL #define STRTAB_STE_1_S1C_CACHE_WT 2UL @@ -251,6 +256,13 @@ #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0) #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32) +#define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0) +#define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6) +#define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8) +#define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10) +#define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12) +#define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14) +#define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16) #define STRTAB_STE_2_S2AA64 (1UL << 51) #define STRTAB_STE_2_S2ENDI (1UL << 52) #define STRTAB_STE_2_S2PTW (1UL << 54) @@ -258,30 +270,34 @@ #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4) -/* Context descriptor (stage-1 only) */ +/* + * Context descriptors. + * + * Linear: when less than 1024 SSIDs are supported + * 2lvl: at most 1024 L1 entries, + * 1024 lazy entries per table. + */ +#define CTXDESC_SPLIT 10 +#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT) + +#define CTXDESC_L1_DESC_DWORDS 1 +#define CTXDESC_L1_DESC_V (1UL << 0) +#define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12) + #define CTXDESC_CD_DWORDS 8 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) -#define ARM64_TCR_T0SZ GENMASK_ULL(5, 0) #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6) -#define ARM64_TCR_TG0 GENMASK_ULL(15, 14) #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) -#define ARM64_TCR_IRGN0 GENMASK_ULL(9, 8) #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) -#define ARM64_TCR_ORGN0 GENMASK_ULL(11, 10) #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12) -#define ARM64_TCR_SH0 GENMASK_ULL(13, 12) #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14) -#define ARM64_TCR_EPD0 (1ULL << 7) #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30) -#define ARM64_TCR_EPD1 (1ULL << 23) #define CTXDESC_CD_0_ENDI (1UL << 15) #define CTXDESC_CD_0_V (1UL << 31) #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32) -#define ARM64_TCR_IPS GENMASK_ULL(34, 32) #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38) -#define ARM64_TCR_TBI0 (1ULL << 37) #define CTXDESC_CD_0_AA64 (1UL << 41) #define CTXDESC_CD_0_S (1UL << 44) @@ -292,9 +308,11 @@ #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4) -/* Convert between AArch64 (CPU) TCR format and SMMU CD format */ -#define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \ - FIELD_GET(ARM64_TCR_##fld, tcr)) +/* + * When the SMMU only supports linear context descriptor tables, pick a + * reasonable size limit (64kB). + */ +#define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3)) /* Command queue */ #define CMDQ_ENT_SZ_SHIFT 4 @@ -323,6 +341,7 @@ #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0) #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12) +#define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12) #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32) #define CMDQ_CFGI_1_LEAF (1UL << 0) #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0) @@ -384,10 +403,6 @@ #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 -/* - * not really modular, but the easiest way to keep compat with existing - * bootargs behaviour is to continue using module_param_named here. - */ static bool disable_bypass = 1; module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO); MODULE_PARM_DESC(disable_bypass, @@ -440,8 +455,11 @@ struct arm_smmu_cmdq_ent { #define CMDQ_OP_CFGI_STE 0x3 #define CMDQ_OP_CFGI_ALL 0x4 + #define CMDQ_OP_CFGI_CD 0x5 + #define CMDQ_OP_CFGI_CD_ALL 0x6 struct { u32 sid; + u32 ssid; union { bool leaf; u8 span; @@ -547,16 +565,30 @@ struct arm_smmu_strtab_l1_desc { dma_addr_t l2ptr_dma; }; +struct arm_smmu_ctx_desc { + u16 asid; + u64 ttbr; + u64 tcr; + u64 mair; +}; + +struct arm_smmu_l1_ctx_desc { + __le64 *l2ptr; + dma_addr_t l2ptr_dma; +}; + +struct arm_smmu_ctx_desc_cfg { + __le64 *cdtab; + dma_addr_t cdtab_dma; + struct arm_smmu_l1_ctx_desc *l1_desc; + unsigned int num_l1_ents; +}; + struct arm_smmu_s1_cfg { - __le64 *cdptr; - dma_addr_t cdptr_dma; - - struct arm_smmu_ctx_desc { - u16 asid; - u64 ttbr; - u64 tcr; - u64 mair; - } cd; + struct arm_smmu_ctx_desc_cfg cdcfg; + struct arm_smmu_ctx_desc cd; + u8 s1fmt; + u8 s1cdmax; }; struct arm_smmu_s2_cfg { @@ -638,6 +670,7 @@ struct arm_smmu_master { u32 *sids; unsigned int num_sids; bool ats_enabled; + unsigned int ssid_bits; }; /* SMMU private data for an IOMMU domain */ @@ -847,15 +880,22 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size); cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK; break; + case CMDQ_OP_CFGI_CD: + cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid); + /* Fallthrough */ case CMDQ_OP_CFGI_STE: cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); break; + case CMDQ_OP_CFGI_CD_ALL: + cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); + break; case CMDQ_OP_CFGI_ALL: /* Cover the entire SID range */ cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); break; case CMDQ_OP_TLBI_NH_VA: + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; @@ -1443,50 +1483,238 @@ static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu) } /* Context descriptor manipulation functions */ -static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr) +static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain, + int ssid, bool leaf) { - u64 val = 0; + size_t i; + unsigned long flags; + struct arm_smmu_master *master; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cmdq_ent cmd = { + .opcode = CMDQ_OP_CFGI_CD, + .cfgi = { + .ssid = ssid, + .leaf = leaf, + }, + }; + + spin_lock_irqsave(&smmu_domain->devices_lock, flags); + list_for_each_entry(master, &smmu_domain->devices, domain_head) { + for (i = 0; i < master->num_sids; i++) { + cmd.cfgi.sid = master->sids[i]; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + } + } + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + + arm_smmu_cmdq_issue_sync(smmu); +} - /* Repack the TCR. Just care about TTBR0 for now */ - val |= ARM_SMMU_TCR2CD(tcr, T0SZ); - val |= ARM_SMMU_TCR2CD(tcr, TG0); - val |= ARM_SMMU_TCR2CD(tcr, IRGN0); - val |= ARM_SMMU_TCR2CD(tcr, ORGN0); - val |= ARM_SMMU_TCR2CD(tcr, SH0); - val |= ARM_SMMU_TCR2CD(tcr, EPD0); - val |= ARM_SMMU_TCR2CD(tcr, EPD1); - val |= ARM_SMMU_TCR2CD(tcr, IPS); +static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu, + struct arm_smmu_l1_ctx_desc *l1_desc) +{ + size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3); - return val; + l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, + &l1_desc->l2ptr_dma, GFP_KERNEL); + if (!l1_desc->l2ptr) { + dev_warn(smmu->dev, + "failed to allocate context descriptor table\n"); + return -ENOMEM; + } + return 0; } -static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu, - struct arm_smmu_s1_cfg *cfg) +static void arm_smmu_write_cd_l1_desc(__le64 *dst, + struct arm_smmu_l1_ctx_desc *l1_desc) { - u64 val; + u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | + CTXDESC_L1_DESC_V; + + WRITE_ONCE(*dst, cpu_to_le64(val)); +} + +static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain, + u32 ssid) +{ + __le64 *l1ptr; + unsigned int idx; + struct arm_smmu_l1_ctx_desc *l1_desc; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg; + if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR) + return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS; + + idx = ssid >> CTXDESC_SPLIT; + l1_desc = &cdcfg->l1_desc[idx]; + if (!l1_desc->l2ptr) { + if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) + return NULL; + + l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS; + arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); + /* An invalid L1CD can be cached */ + arm_smmu_sync_cd(smmu_domain, ssid, false); + } + idx = ssid & (CTXDESC_L2_ENTRIES - 1); + return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS; +} + +static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, + int ssid, struct arm_smmu_ctx_desc *cd) +{ /* - * We don't need to issue any invalidation here, as we'll invalidate - * the STE when installing the new entry anyway. + * This function handles the following cases: + * + * (1) Install primary CD, for normal DMA traffic (SSID = 0). + * (2) Install a secondary CD, for SID+SSID traffic. + * (3) Update ASID of a CD. Atomically write the first 64 bits of the + * CD, then invalidate the old entry and mappings. + * (4) Remove a secondary CD. */ - val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) | + u64 val; + bool cd_live; + __le64 *cdptr; + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax))) + return -E2BIG; + + cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid); + if (!cdptr) + return -ENOMEM; + + val = le64_to_cpu(cdptr[0]); + cd_live = !!(val & CTXDESC_CD_0_V); + + if (!cd) { /* (4) */ + val = 0; + } else if (cd_live) { /* (3) */ + val &= ~CTXDESC_CD_0_ASID; + val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid); + /* + * Until CD+TLB invalidation, both ASIDs may be used for tagging + * this substream's traffic + */ + } else { /* (1) and (2) */ + cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); + cdptr[2] = 0; + cdptr[3] = cpu_to_le64(cd->mair); + + /* + * STE is live, and the SMMU might read dwords of this CD in any + * order. Ensure that it observes valid values before reading + * V=1. + */ + arm_smmu_sync_cd(smmu_domain, ssid, true); + + val = cd->tcr | #ifdef __BIG_ENDIAN - CTXDESC_CD_0_ENDI | + CTXDESC_CD_0_ENDI | #endif - CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET | - CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) | - CTXDESC_CD_0_V; + CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET | + CTXDESC_CD_0_AA64 | + FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) | + CTXDESC_CD_0_V; - /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */ - if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE) - val |= CTXDESC_CD_0_S; + /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */ + if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE) + val |= CTXDESC_CD_0_S; + } - cfg->cdptr[0] = cpu_to_le64(val); + /* + * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3 + * "Configuration structures and configuration invalidation completion" + * + * The size of single-copy atomic reads made by the SMMU is + * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single + * field within an aligned 64-bit span of a structure can be altered + * without first making the structure invalid. + */ + WRITE_ONCE(cdptr[0], cpu_to_le64(val)); + arm_smmu_sync_cd(smmu_domain, ssid, true); + return 0; +} + +static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain) +{ + int ret; + size_t l1size; + size_t max_contexts; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; + struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg; + + max_contexts = 1 << cfg->s1cdmax; + + if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || + max_contexts <= CTXDESC_L2_ENTRIES) { + cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR; + cdcfg->num_l1_ents = max_contexts; - val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK; - cfg->cdptr[1] = cpu_to_le64(val); + l1size = max_contexts * (CTXDESC_CD_DWORDS << 3); + } else { + cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2; + cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts, + CTXDESC_L2_ENTRIES); + + cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents, + sizeof(*cdcfg->l1_desc), + GFP_KERNEL); + if (!cdcfg->l1_desc) + return -ENOMEM; + + l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); + } + + cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma, + GFP_KERNEL); + if (!cdcfg->cdtab) { + dev_warn(smmu->dev, "failed to allocate context descriptor\n"); + ret = -ENOMEM; + goto err_free_l1; + } + + return 0; - cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair); +err_free_l1: + if (cdcfg->l1_desc) { + devm_kfree(smmu->dev, cdcfg->l1_desc); + cdcfg->l1_desc = NULL; + } + return ret; +} + +static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain) +{ + int i; + size_t size, l1size; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg; + + if (cdcfg->l1_desc) { + size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3); + + for (i = 0; i < cdcfg->num_l1_ents; i++) { + if (!cdcfg->l1_desc[i].l2ptr) + continue; + + dmam_free_coherent(smmu->dev, size, + cdcfg->l1_desc[i].l2ptr, + cdcfg->l1_desc[i].l2ptr_dma); + } + devm_kfree(smmu->dev, cdcfg->l1_desc); + cdcfg->l1_desc = NULL; + + l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); + } else { + l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3); + } + + dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma); + cdcfg->cdtab_dma = 0; + cdcfg->cdtab = NULL; } /* Stream table manipulation functions */ @@ -1608,6 +1836,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, if (s1_cfg) { BUG_ON(ste_live); dst[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | @@ -1617,8 +1846,10 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); - val |= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) | - FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS); + val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | + FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) | + FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt); } if (s2_cfg) { @@ -1642,7 +1873,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, STRTAB_STE_1_EATS_TRANS)); arm_smmu_sync_ste_for_sid(smmu, sid); - dst[0] = cpu_to_le64(val); + /* See comment in arm_smmu_write_ctx_desc() */ + WRITE_ONCE(dst[0], cpu_to_le64(val)); arm_smmu_sync_ste_for_sid(smmu, sid); /* It's likely that we'll want to use the new STE soon */ @@ -1675,7 +1907,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) desc->span = STRTAB_SPLIT + 1; desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma, - GFP_KERNEL | __GFP_ZERO); + GFP_KERNEL); if (!desc->l2ptr) { dev_err(smmu->dev, "failed to allocate l2 stream table for SID %u\n", @@ -2131,12 +2363,8 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; - if (cfg->cdptr) { - dmam_free_coherent(smmu_domain->smmu->dev, - CTXDESC_CD_DWORDS << 3, - cfg->cdptr, - cfg->cdptr_dma); - + if (cfg->cdcfg.cdtab) { + arm_smmu_free_cd_tables(smmu_domain); arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid); } } else { @@ -2149,55 +2377,82 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) } static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_master *master, struct io_pgtable_cfg *pgtbl_cfg) { int ret; int asid; struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; + typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits); if (asid < 0) return asid; - cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3, - &cfg->cdptr_dma, - GFP_KERNEL | __GFP_ZERO); - if (!cfg->cdptr) { - dev_warn(smmu->dev, "failed to allocate context descriptor\n"); - ret = -ENOMEM; + cfg->s1cdmax = master->ssid_bits; + + ret = arm_smmu_alloc_cd_tables(smmu_domain); + if (ret) goto out_free_asid; - } cfg->cd.asid = (u16)asid; - cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; - cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr; + cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; + cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | + FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | + FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | + CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair; + + /* + * Note that this will end up calling arm_smmu_sync_cd() before + * the master has been added to the devices list for this domain. + * This isn't an issue because the STE hasn't been installed yet. + */ + ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd); + if (ret) + goto out_free_cd_tables; + return 0; +out_free_cd_tables: + arm_smmu_free_cd_tables(smmu_domain); out_free_asid: arm_smmu_bitmap_free(smmu->asid_map, asid); return ret; } static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_master *master, struct io_pgtable_cfg *pgtbl_cfg) { int vmid; struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; + typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr; vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits); if (vmid < 0) return vmid; + vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr; cfg->vmid = (u16)vmid; cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; - cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; + cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps); return 0; } -static int arm_smmu_domain_finalise(struct iommu_domain *domain) +static int arm_smmu_domain_finalise(struct iommu_domain *domain, + struct arm_smmu_master *master) { int ret; unsigned long ias, oas; @@ -2205,6 +2460,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable_ops *pgtbl_ops; int (*finalise_stage_fn)(struct arm_smmu_domain *, + struct arm_smmu_master *, struct io_pgtable_cfg *); struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; @@ -2259,7 +2515,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; domain->geometry.force_aperture = true; - ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); + ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg); if (ret < 0) { free_io_pgtable_ops(pgtbl_ops); return ret; @@ -2412,7 +2668,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (!smmu_domain->smmu) { smmu_domain->smmu = smmu; - ret = arm_smmu_domain_finalise(domain); + ret = arm_smmu_domain_finalise(domain, master); if (ret) { smmu_domain->smmu = NULL; goto out_unlock; @@ -2424,6 +2680,13 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) dev_name(smmu->dev)); ret = -ENXIO; goto out_unlock; + } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && + master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) { + dev_err(dev, + "cannot attach to incompatible domain (%u SSID bits != %u)\n", + smmu_domain->s1_cfg.s1cdmax, master->ssid_bits); + ret = -EINVAL; + goto out_unlock; } master->domain = smmu_domain; @@ -2431,9 +2694,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) master->ats_enabled = arm_smmu_ats_supported(master); - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) - arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg); - arm_smmu_install_ste_for_dev(master); spin_lock_irqsave(&smmu_domain->devices_lock, flags); @@ -2534,51 +2794,66 @@ static int arm_smmu_add_device(struct device *dev) if (!fwspec || fwspec->ops != &arm_smmu_ops) return -ENODEV; - /* - * We _can_ actually withstand dodgy bus code re-calling add_device() - * without an intervening remove_device()/of_xlate() sequence, but - * we're not going to do so quietly... - */ - if (WARN_ON_ONCE(fwspec->iommu_priv)) { - master = fwspec->iommu_priv; - smmu = master->smmu; - } else { - smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); - if (!smmu) - return -ENODEV; - master = kzalloc(sizeof(*master), GFP_KERNEL); - if (!master) - return -ENOMEM; - master->dev = dev; - master->smmu = smmu; - master->sids = fwspec->ids; - master->num_sids = fwspec->num_ids; - fwspec->iommu_priv = master; - } + if (WARN_ON_ONCE(fwspec->iommu_priv)) + return -EBUSY; + + smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); + if (!smmu) + return -ENODEV; + + master = kzalloc(sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + + master->dev = dev; + master->smmu = smmu; + master->sids = fwspec->ids; + master->num_sids = fwspec->num_ids; + fwspec->iommu_priv = master; /* Check the SIDs are in range of the SMMU and our stream table */ for (i = 0; i < master->num_sids; i++) { u32 sid = master->sids[i]; - if (!arm_smmu_sid_in_range(smmu, sid)) - return -ERANGE; + if (!arm_smmu_sid_in_range(smmu, sid)) { + ret = -ERANGE; + goto err_free_master; + } /* Ensure l2 strtab is initialised */ if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { ret = arm_smmu_init_l2_strtab(smmu, sid); if (ret) - return ret; + goto err_free_master; } } + master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits); + + if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB)) + master->ssid_bits = min_t(u8, master->ssid_bits, + CTXDESC_LINEAR_CDMAX); + + ret = iommu_device_link(&smmu->iommu, dev); + if (ret) + goto err_free_master; + group = iommu_group_get_for_dev(dev); - if (!IS_ERR(group)) { - iommu_group_put(group); - iommu_device_link(&smmu->iommu, dev); + if (IS_ERR(group)) { + ret = PTR_ERR(group); + goto err_unlink; } - return PTR_ERR_OR_ZERO(group); + iommu_group_put(group); + return 0; + +err_unlink: + iommu_device_unlink(&smmu->iommu, dev); +err_free_master: + kfree(master); + fwspec->iommu_priv = NULL; + return ret; } static void arm_smmu_remove_device(struct device *dev) @@ -2710,15 +2985,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, iommu_dma_get_resv_regions(dev, head); } -static void arm_smmu_put_resv_regions(struct device *dev, - struct list_head *head) -{ - struct iommu_resv_region *entry, *next; - - list_for_each_entry_safe(entry, next, head, list) - kfree(entry); -} - static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, @@ -2736,7 +3002,7 @@ static struct iommu_ops arm_smmu_ops = { .domain_set_attr = arm_smmu_domain_set_attr, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, - .put_resv_regions = arm_smmu_put_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, .pgsize_bitmap = -1UL, /* Restricted during device attach */ }; @@ -2883,7 +3149,7 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3); strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, - GFP_KERNEL | __GFP_ZERO); + GFP_KERNEL); if (!strtab) { dev_err(smmu->dev, "failed to allocate l1 stream table (%u bytes)\n", @@ -2910,7 +3176,7 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3); strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma, - GFP_KERNEL | __GFP_ZERO); + GFP_KERNEL); if (!strtab) { dev_err(smmu->dev, "failed to allocate linear stream table (%u bytes)\n", @@ -3570,6 +3836,43 @@ static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) return SZ_128K; } +static int arm_smmu_set_bus_ops(struct iommu_ops *ops) +{ + int err; + +#ifdef CONFIG_PCI + if (pci_bus_type.iommu_ops != ops) { + err = bus_set_iommu(&pci_bus_type, ops); + if (err) + return err; + } +#endif +#ifdef CONFIG_ARM_AMBA + if (amba_bustype.iommu_ops != ops) { + err = bus_set_iommu(&amba_bustype, ops); + if (err) + goto err_reset_pci_ops; + } +#endif + if (platform_bus_type.iommu_ops != ops) { + err = bus_set_iommu(&platform_bus_type, ops); + if (err) + goto err_reset_amba_ops; + } + + return 0; + +err_reset_amba_ops: +#ifdef CONFIG_ARM_AMBA + bus_set_iommu(&amba_bustype, NULL); +#endif +err_reset_pci_ops: __maybe_unused; +#ifdef CONFIG_PCI + bus_set_iommu(&pci_bus_type, NULL); +#endif + return err; +} + static int arm_smmu_device_probe(struct platform_device *pdev) { int irq, ret; @@ -3599,7 +3902,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) /* Base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) { + if (resource_size(res) < arm_smmu_resource_size(smmu)) { dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; } @@ -3660,48 +3963,45 @@ static int arm_smmu_device_probe(struct platform_device *pdev) return ret; } -#ifdef CONFIG_PCI - if (pci_bus_type.iommu_ops != &arm_smmu_ops) { - pci_request_acs(); - ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops); - if (ret) - return ret; - } -#endif -#ifdef CONFIG_ARM_AMBA - if (amba_bustype.iommu_ops != &arm_smmu_ops) { - ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops); - if (ret) - return ret; - } -#endif - if (platform_bus_type.iommu_ops != &arm_smmu_ops) { - ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops); - if (ret) - return ret; - } - return 0; + return arm_smmu_set_bus_ops(&arm_smmu_ops); } -static void arm_smmu_device_shutdown(struct platform_device *pdev) +static int arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); + arm_smmu_set_bus_ops(NULL); + iommu_device_unregister(&smmu->iommu); + iommu_device_sysfs_remove(&smmu->iommu); arm_smmu_device_disable(smmu); + + return 0; +} + +static void arm_smmu_device_shutdown(struct platform_device *pdev) +{ + arm_smmu_device_remove(pdev); } static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v3", }, { }, }; +MODULE_DEVICE_TABLE(of, arm_smmu_of_match); static struct platform_driver arm_smmu_driver = { .driver = { - .name = "arm-smmu-v3", - .of_match_table = of_match_ptr(arm_smmu_of_match), - .suppress_bind_attrs = true, + .name = "arm-smmu-v3", + .of_match_table = arm_smmu_of_match, + .suppress_bind_attrs = true, }, .probe = arm_smmu_device_probe, + .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -builtin_platform_driver(arm_smmu_driver); +module_platform_driver(arm_smmu_driver); + +MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations"); +MODULE_AUTHOR("Will Deacon <will@kernel.org>"); +MODULE_ALIAS("platform:arm-smmu-v3"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 4f1a350d9529..16c4b87af42b 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -27,8 +27,7 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> -#include <linux/init.h> -#include <linux/moduleparam.h> +#include <linux/module.h> #include <linux/of.h> #include <linux/of_address.h> #include <linux/of_device.h> @@ -60,10 +59,6 @@ #define MSI_IOVA_LENGTH 0x100000 static int force_stage; -/* - * not really modular, but the easiest way to keep compat with existing - * bootargs behaviour is to continue using module_param() here. - */ module_param(force_stage, int, S_IRUGO); MODULE_PARM_DESC(force_stage, "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); @@ -131,6 +126,12 @@ static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) return container_of(dom, struct arm_smmu_domain, domain); } +static struct platform_driver arm_smmu_driver; +static struct iommu_ops arm_smmu_ops; + +#ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS +static int arm_smmu_bus_init(struct iommu_ops *ops); + static struct device_node *dev_get_dev_node(struct device *dev) { if (dev_is_pci(dev)) { @@ -166,9 +167,6 @@ static int __find_legacy_master_phandle(struct device *dev, void *data) return err == -ENOENT ? 0 : err; } -static struct platform_driver arm_smmu_driver; -static struct iommu_ops arm_smmu_ops; - static int arm_smmu_register_legacy_master(struct device *dev, struct arm_smmu_device **smmu) { @@ -220,6 +218,27 @@ static int arm_smmu_register_legacy_master(struct device *dev, return err; } +/* + * With the legacy DT binding in play, we have no guarantees about + * probe order, but then we're also not doing default domains, so we can + * delay setting bus ops until we're sure every possible SMMU is ready, + * and that way ensure that no add_device() calls get missed. + */ +static int arm_smmu_legacy_bus_init(void) +{ + if (using_legacy_binding) + return arm_smmu_bus_init(&arm_smmu_ops); + return 0; +} +device_initcall_sync(arm_smmu_legacy_bus_init); +#else +static int arm_smmu_register_legacy_master(struct device *dev, + struct arm_smmu_device **smmu) +{ + return -ENODEV; +} +#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ + static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) { int idx; @@ -252,7 +271,7 @@ static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { reg = arm_smmu_readl(smmu, page, status); - if (!(reg & sTLBGSTATUS_GSACTIVE)) + if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) return; cpu_relax(); } @@ -459,7 +478,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) int idx = smmu_domain->cfg.cbndx; fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (!(fsr & FSR_FAULT)) + if (!(fsr & ARM_SMMU_FSR_FAULT)) return IRQ_NONE; fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); @@ -491,7 +510,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) if (__ratelimit(&rs)) { if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && - (gfsr & sGFSR_USF)) + (gfsr & ARM_SMMU_sGFSR_USF)) dev_err(smmu->dev, "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n", (u16)gfsynr1); @@ -521,26 +540,28 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; } else { - cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr; - cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; - cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM); + cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg); + cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg); if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) - cb->tcr[1] |= TCR2_AS; + cb->tcr[1] |= ARM_SMMU_TCR2_AS; + else + cb->tcr[0] |= ARM_SMMU_TCR_EAE; } } else { - cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; + cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg); } /* TTBRs */ if (stage1) { if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { - cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0]; - cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1]; + cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr; + cb->ttbr[1] = 0; } else { - cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; - cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid); - cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; - cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid); + cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; + cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, + cfg->asid); + cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, + cfg->asid); } } else { cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; @@ -576,31 +597,33 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) /* CBA2R */ if (smmu->version > ARM_SMMU_V1) { if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) - reg = CBA2R_VA64; + reg = ARM_SMMU_CBA2R_VA64; else reg = 0; /* 16-bit VMIDs live in CBA2R */ if (smmu->features & ARM_SMMU_FEAT_VMID16) - reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid); + reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid); arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg); } /* CBAR */ - reg = FIELD_PREP(CBAR_TYPE, cfg->cbar); + reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar); if (smmu->version < ARM_SMMU_V2) - reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx); + reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx); /* * Use the weakest shareability/memory types, so they are * overridden by the ttbcr/pte. */ if (stage1) { - reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) | - FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB); + reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG, + ARM_SMMU_CBAR_S1_BPSHCFG_NSH) | + FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR, + ARM_SMMU_CBAR_S1_MEMATTR_WB); } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { /* 8-bit VMIDs live in CBAR */ - reg |= FIELD_PREP(CBAR_VMID, cfg->vmid); + reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid); } arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg); @@ -632,11 +655,12 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) } /* SCTLR */ - reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M; + reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | + ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; if (stage1) - reg |= SCTLR_S1_ASIDPNE; + reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) - reg |= SCTLR_E; + reg |= ARM_SMMU_SCTLR_E; arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); } @@ -818,7 +842,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); - cfg->irptndx = INVALID_IRPTNDX; + cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX; } mutex_unlock(&smmu_domain->init_mutex); @@ -856,7 +880,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) smmu->cbs[cfg->cbndx].cfg = NULL; arm_smmu_write_context_bank(smmu, cfg->cbndx); - if (cfg->irptndx != INVALID_IRPTNDX) { + if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) { irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; devm_free_irq(smmu->dev, irq, domain); } @@ -912,23 +936,24 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) { struct arm_smmu_smr *smr = smmu->smrs + idx; - u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask); + u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) | + FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask); if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) - reg |= SMR_VALID; + reg |= ARM_SMMU_SMR_VALID; arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg); } static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) { struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; - u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) | - FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) | - FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg); + u32 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | + FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | + FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && smmu->smrs[idx].valid) - reg |= S2CR_EXIDVALID; + reg |= ARM_SMMU_S2CR_EXIDVALID; arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); } @@ -946,24 +971,37 @@ static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) { u32 smr; + int i; if (!smmu->smrs) return; - + /* + * If we've had to accommodate firmware memory regions, we may + * have live SMRs by now; tread carefully... + * + * Somewhat perversely, not having a free SMR for this test implies we + * can get away without it anyway, as we'll only be able to 'allocate' + * these SMRs for the ID/mask values we're already trusting to be OK. + */ + for (i = 0; i < smmu->num_mapping_groups; i++) + if (!smmu->smrs[i].valid) + goto smr_ok; + return; +smr_ok: /* * SMR.ID bits may not be preserved if the corresponding MASK * bits are set, so check each one separately. We can reject * masters later if they try to claim IDs outside these masks. */ - smr = FIELD_PREP(SMR_ID, smmu->streamid_mask); - arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr); - smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0)); - smmu->streamid_mask = FIELD_GET(SMR_ID, smr); + smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask); + arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); + smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); + smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr); - smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask); - arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr); - smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0)); - smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr); + smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask); + arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); + smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); + smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); } static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) @@ -1032,8 +1070,8 @@ static int arm_smmu_master_alloc_smes(struct device *dev) mutex_lock(&smmu->stream_map_mutex); /* Figure out a viable stream map entry allocation */ for_each_cfg_sme(fwspec, i, idx) { - u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]); - u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]); + u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); + u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); if (idx != INVALID_SMENDX) { ret = -EEXIST; @@ -1277,7 +1315,8 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; - if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) { + if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE), + 5, 50)) { spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); dev_err(dev, "iova to phys timed out on %pad. Falling back to software table walk.\n", @@ -1287,7 +1326,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR); spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); - if (phys & CB_PAR_F) { + if (phys & ARM_SMMU_CB_PAR_F) { dev_err(dev, "translation fault!\n"); dev_err(dev, "PAR = 0x%llx\n", phys); return 0; @@ -1368,8 +1407,8 @@ static int arm_smmu_add_device(struct device *dev) ret = -EINVAL; for (i = 0; i < fwspec->num_ids; i++) { - u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]); - u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]); + u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); + u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); if (sid & ~smmu->streamid_mask) { dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", @@ -1550,12 +1589,12 @@ static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) u32 mask, fwid = 0; if (args->args_count > 0) - fwid |= FIELD_PREP(SMR_ID, args->args[0]); + fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]); if (args->args_count > 1) - fwid |= FIELD_PREP(SMR_MASK, args->args[1]); + fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]); else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) - fwid |= FIELD_PREP(SMR_MASK, mask); + fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask); return iommu_fwspec_add_ids(dev, &fwid, 1); } @@ -1576,15 +1615,6 @@ static void arm_smmu_get_resv_regions(struct device *dev, iommu_dma_get_resv_regions(dev, head); } -static void arm_smmu_put_resv_regions(struct device *dev, - struct list_head *head) -{ - struct iommu_resv_region *entry, *next; - - list_for_each_entry_safe(entry, next, head, list) - kfree(entry); -} - static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, @@ -1602,7 +1632,7 @@ static struct iommu_ops arm_smmu_ops = { .domain_set_attr = arm_smmu_domain_set_attr, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, - .put_resv_regions = arm_smmu_put_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, .pgsize_bitmap = -1UL, /* Restricted during device attach */ }; @@ -1625,7 +1655,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Make sure all context banks are disabled and clear CB_FSR */ for (i = 0; i < smmu->num_context_banks; ++i) { arm_smmu_write_context_bank(smmu, i); - arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT); + arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT); } /* Invalidate the TLB, just in case */ @@ -1635,29 +1665,30 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); /* Enable fault reporting */ - reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE); + reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE | + ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE); /* Disable TLB broadcasting. */ - reg |= (sCR0_VMIDPNE | sCR0_PTM); + reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM); /* Enable client access, handling unmatched streams as appropriate */ - reg &= ~sCR0_CLIENTPD; + reg &= ~ARM_SMMU_sCR0_CLIENTPD; if (disable_bypass) - reg |= sCR0_USFCFG; + reg |= ARM_SMMU_sCR0_USFCFG; else - reg &= ~sCR0_USFCFG; + reg &= ~ARM_SMMU_sCR0_USFCFG; /* Disable forced broadcasting */ - reg &= ~sCR0_FB; + reg &= ~ARM_SMMU_sCR0_FB; /* Don't upgrade barriers */ - reg &= ~(sCR0_BSU); + reg &= ~(ARM_SMMU_sCR0_BSU); if (smmu->features & ARM_SMMU_FEAT_VMID16) - reg |= sCR0_VMID16EN; + reg |= ARM_SMMU_sCR0_VMID16EN; if (smmu->features & ARM_SMMU_FEAT_EXIDS) - reg |= sCR0_EXIDENABLE; + reg |= ARM_SMMU_sCR0_EXIDENABLE; if (smmu->impl && smmu->impl->reset) smmu->impl->reset(smmu); @@ -1702,21 +1733,21 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) /* Restrict available stages based on module parameter */ if (force_stage == 1) - id &= ~(ID0_S2TS | ID0_NTS); + id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS); else if (force_stage == 2) - id &= ~(ID0_S1TS | ID0_NTS); + id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS); - if (id & ID0_S1TS) { + if (id & ARM_SMMU_ID0_S1TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S1; dev_notice(smmu->dev, "\tstage 1 translation\n"); } - if (id & ID0_S2TS) { + if (id & ARM_SMMU_ID0_S2TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S2; dev_notice(smmu->dev, "\tstage 2 translation\n"); } - if (id & ID0_NTS) { + if (id & ARM_SMMU_ID0_NTS) { smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; dev_notice(smmu->dev, "\tnested translation\n"); } @@ -1727,8 +1758,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return -ENODEV; } - if ((id & ID0_S1TS) && - ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) { + if ((id & ARM_SMMU_ID0_S1TS) && + ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) { smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; dev_notice(smmu->dev, "\taddress translation ops\n"); } @@ -1739,7 +1770,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) * Fortunately, this also opens up a workaround for systems where the * ID register value has ended up configured incorrectly. */ - cttw_reg = !!(id & ID0_CTTW); + cttw_reg = !!(id & ARM_SMMU_ID0_CTTW); if (cttw_fw || cttw_reg) dev_notice(smmu->dev, "\t%scoherent table walk\n", cttw_fw ? "" : "non-"); @@ -1748,16 +1779,16 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) "\t(IDR0.CTTW overridden by FW configuration)\n"); /* Max. number of entries we have for stream matching/indexing */ - if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) { + if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) { smmu->features |= ARM_SMMU_FEAT_EXIDS; size = 1 << 16; } else { - size = 1 << FIELD_GET(ID0_NUMSIDB, id); + size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id); } smmu->streamid_mask = size - 1; - if (id & ID0_SMS) { + if (id & ARM_SMMU_ID0_SMS) { smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; - size = FIELD_GET(ID0_NUMSMRG, id); + size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id); if (size == 0) { dev_err(smmu->dev, "stream-matching supported, but no SMRs present!\n"); @@ -1785,18 +1816,19 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) mutex_init(&smmu->stream_map_mutex); spin_lock_init(&smmu->global_sync_lock); - if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { + if (smmu->version < ARM_SMMU_V2 || + !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) { smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; - if (!(id & ID0_PTFS_NO_AARCH32S)) + if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S)) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; } /* ID1 */ id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1); - smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; + smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12; /* Check for size mismatch of SMMU address space from mapped region */ - size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1); + size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1); if (smmu->numpage != 2 * size << smmu->pgshift) dev_warn(smmu->dev, "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n", @@ -1804,8 +1836,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */ smmu->numpage = size; - smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id); - smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id); + smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id); + smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id); if (smmu->num_s2_context_banks > smmu->num_context_banks) { dev_err(smmu->dev, "impossible number of S2 context banks!\n"); return -ENODEV; @@ -1819,14 +1851,14 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) /* ID2 */ id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2); - size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id)); + size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id)); smmu->ipa_size = size; /* The output mask is also applied for bypass */ - size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id)); + size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id)); smmu->pa_size = size; - if (id & ID2_VMID16) + if (id & ARM_SMMU_ID2_VMID16) smmu->features |= ARM_SMMU_FEAT_VMID16; /* @@ -1843,13 +1875,13 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if (smmu->version == ARM_SMMU_V1_64K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } else { - size = FIELD_GET(ID2_UBS, id); + size = FIELD_GET(ARM_SMMU_ID2_UBS, id); smmu->va_size = arm_smmu_id_size_to_bits(size); - if (id & ID2_PTFS_4K) + if (id & ARM_SMMU_ID2_PTFS_4K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; - if (id & ID2_PTFS_16K) + if (id & ARM_SMMU_ID2_PTFS_16K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; - if (id & ID2_PTFS_64K) + if (id & ARM_SMMU_ID2_PTFS_64K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } @@ -1911,6 +1943,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, { }, }; +MODULE_DEVICE_TABLE(of, arm_smmu_of_match); #ifdef CONFIG_ACPI static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) @@ -1997,8 +2030,10 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); if (legacy_binding && !using_generic_binding) { - if (!using_legacy_binding) - pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n"); + if (!using_legacy_binding) { + pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n", + IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU"); + } using_legacy_binding = true; } else if (!legacy_binding && !using_legacy_binding) { using_generic_binding = true; @@ -2013,25 +2048,50 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, return 0; } -static void arm_smmu_bus_init(void) +static int arm_smmu_bus_init(struct iommu_ops *ops) { + int err; + /* Oh, for a proper bus abstraction */ - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &arm_smmu_ops); + if (!iommu_present(&platform_bus_type)) { + err = bus_set_iommu(&platform_bus_type, ops); + if (err) + return err; + } #ifdef CONFIG_ARM_AMBA - if (!iommu_present(&amba_bustype)) - bus_set_iommu(&amba_bustype, &arm_smmu_ops); + if (!iommu_present(&amba_bustype)) { + err = bus_set_iommu(&amba_bustype, ops); + if (err) + goto err_reset_platform_ops; + } #endif #ifdef CONFIG_PCI if (!iommu_present(&pci_bus_type)) { - pci_request_acs(); - bus_set_iommu(&pci_bus_type, &arm_smmu_ops); + err = bus_set_iommu(&pci_bus_type, ops); + if (err) + goto err_reset_amba_ops; } #endif #ifdef CONFIG_FSL_MC_BUS - if (!iommu_present(&fsl_mc_bus_type)) - bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops); + if (!iommu_present(&fsl_mc_bus_type)) { + err = bus_set_iommu(&fsl_mc_bus_type, ops); + if (err) + goto err_reset_pci_ops; + } #endif + return 0; + +err_reset_pci_ops: __maybe_unused; +#ifdef CONFIG_PCI + bus_set_iommu(&pci_bus_type, NULL); +#endif +err_reset_amba_ops: __maybe_unused; +#ifdef CONFIG_ARM_AMBA + bus_set_iommu(&amba_bustype, NULL); +#endif +err_reset_platform_ops: __maybe_unused; + bus_set_iommu(&platform_bus_type, NULL); + return err; } static int arm_smmu_device_probe(struct platform_device *pdev) @@ -2177,38 +2237,28 @@ static int arm_smmu_device_probe(struct platform_device *pdev) * ready to handle default domain setup as soon as any SMMU exists. */ if (!using_legacy_binding) - arm_smmu_bus_init(); + return arm_smmu_bus_init(&arm_smmu_ops); return 0; } -/* - * With the legacy DT binding in play, though, we have no guarantees about - * probe order, but then we're also not doing default domains, so we can - * delay setting bus ops until we're sure every possible SMMU is ready, - * and that way ensure that no add_device() calls get missed. - */ -static int arm_smmu_legacy_bus_init(void) -{ - if (using_legacy_binding) - arm_smmu_bus_init(); - return 0; -} -device_initcall_sync(arm_smmu_legacy_bus_init); - -static void arm_smmu_device_shutdown(struct platform_device *pdev) +static int arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); if (!smmu) - return; + return -ENODEV; if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_err(&pdev->dev, "removing device with active domains!\n"); + arm_smmu_bus_init(NULL); + iommu_device_unregister(&smmu->iommu); + iommu_device_sysfs_remove(&smmu->iommu); + arm_smmu_rpm_get(smmu); /* Turn the thing off */ - arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD); + arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); arm_smmu_rpm_put(smmu); if (pm_runtime_enabled(smmu->dev)) @@ -2217,6 +2267,12 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) clk_bulk_disable(smmu->num_clks, smmu->clks); clk_bulk_unprepare(smmu->num_clks, smmu->clks); + return 0; +} + +static void arm_smmu_device_shutdown(struct platform_device *pdev) +{ + arm_smmu_device_remove(pdev); } static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) @@ -2267,11 +2323,17 @@ static const struct dev_pm_ops arm_smmu_pm_ops = { static struct platform_driver arm_smmu_driver = { .driver = { .name = "arm-smmu", - .of_match_table = of_match_ptr(arm_smmu_of_match), + .of_match_table = arm_smmu_of_match, .pm = &arm_smmu_pm_ops, - .suppress_bind_attrs = true, + .suppress_bind_attrs = true, }, .probe = arm_smmu_device_probe, + .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -builtin_platform_driver(arm_smmu_driver); +module_platform_driver(arm_smmu_driver); + +MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); +MODULE_AUTHOR("Will Deacon <will@kernel.org>"); +MODULE_ALIAS("platform:arm-smmu"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index 62b9f0cec49b..8d1cd54d82a6 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -11,6 +11,7 @@ #define _ARM_SMMU_H #include <linux/atomic.h> +#include <linux/bitfield.h> #include <linux/bits.h> #include <linux/clk.h> #include <linux/device.h> @@ -23,51 +24,51 @@ /* Configuration registers */ #define ARM_SMMU_GR0_sCR0 0x0 -#define sCR0_VMID16EN BIT(31) -#define sCR0_BSU GENMASK(15, 14) -#define sCR0_FB BIT(13) -#define sCR0_PTM BIT(12) -#define sCR0_VMIDPNE BIT(11) -#define sCR0_USFCFG BIT(10) -#define sCR0_GCFGFIE BIT(5) -#define sCR0_GCFGFRE BIT(4) -#define sCR0_EXIDENABLE BIT(3) -#define sCR0_GFIE BIT(2) -#define sCR0_GFRE BIT(1) -#define sCR0_CLIENTPD BIT(0) +#define ARM_SMMU_sCR0_VMID16EN BIT(31) +#define ARM_SMMU_sCR0_BSU GENMASK(15, 14) +#define ARM_SMMU_sCR0_FB BIT(13) +#define ARM_SMMU_sCR0_PTM BIT(12) +#define ARM_SMMU_sCR0_VMIDPNE BIT(11) +#define ARM_SMMU_sCR0_USFCFG BIT(10) +#define ARM_SMMU_sCR0_GCFGFIE BIT(5) +#define ARM_SMMU_sCR0_GCFGFRE BIT(4) +#define ARM_SMMU_sCR0_EXIDENABLE BIT(3) +#define ARM_SMMU_sCR0_GFIE BIT(2) +#define ARM_SMMU_sCR0_GFRE BIT(1) +#define ARM_SMMU_sCR0_CLIENTPD BIT(0) /* Auxiliary Configuration register */ #define ARM_SMMU_GR0_sACR 0x10 /* Identification registers */ #define ARM_SMMU_GR0_ID0 0x20 -#define ID0_S1TS BIT(30) -#define ID0_S2TS BIT(29) -#define ID0_NTS BIT(28) -#define ID0_SMS BIT(27) -#define ID0_ATOSNS BIT(26) -#define ID0_PTFS_NO_AARCH32 BIT(25) -#define ID0_PTFS_NO_AARCH32S BIT(24) -#define ID0_NUMIRPT GENMASK(23, 16) -#define ID0_CTTW BIT(14) -#define ID0_NUMSIDB GENMASK(12, 9) -#define ID0_EXIDS BIT(8) -#define ID0_NUMSMRG GENMASK(7, 0) +#define ARM_SMMU_ID0_S1TS BIT(30) +#define ARM_SMMU_ID0_S2TS BIT(29) +#define ARM_SMMU_ID0_NTS BIT(28) +#define ARM_SMMU_ID0_SMS BIT(27) +#define ARM_SMMU_ID0_ATOSNS BIT(26) +#define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25) +#define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24) +#define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16) +#define ARM_SMMU_ID0_CTTW BIT(14) +#define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9) +#define ARM_SMMU_ID0_EXIDS BIT(8) +#define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0) #define ARM_SMMU_GR0_ID1 0x24 -#define ID1_PAGESIZE BIT(31) -#define ID1_NUMPAGENDXB GENMASK(30, 28) -#define ID1_NUMS2CB GENMASK(23, 16) -#define ID1_NUMCB GENMASK(7, 0) +#define ARM_SMMU_ID1_PAGESIZE BIT(31) +#define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28) +#define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16) +#define ARM_SMMU_ID1_NUMCB GENMASK(7, 0) #define ARM_SMMU_GR0_ID2 0x28 -#define ID2_VMID16 BIT(15) -#define ID2_PTFS_64K BIT(14) -#define ID2_PTFS_16K BIT(13) -#define ID2_PTFS_4K BIT(12) -#define ID2_UBS GENMASK(11, 8) -#define ID2_OAS GENMASK(7, 4) -#define ID2_IAS GENMASK(3, 0) +#define ARM_SMMU_ID2_VMID16 BIT(15) +#define ARM_SMMU_ID2_PTFS_64K BIT(14) +#define ARM_SMMU_ID2_PTFS_16K BIT(13) +#define ARM_SMMU_ID2_PTFS_4K BIT(12) +#define ARM_SMMU_ID2_UBS GENMASK(11, 8) +#define ARM_SMMU_ID2_OAS GENMASK(7, 4) +#define ARM_SMMU_ID2_IAS GENMASK(3, 0) #define ARM_SMMU_GR0_ID3 0x2c #define ARM_SMMU_GR0_ID4 0x30 @@ -75,11 +76,11 @@ #define ARM_SMMU_GR0_ID6 0x38 #define ARM_SMMU_GR0_ID7 0x3c -#define ID7_MAJOR GENMASK(7, 4) -#define ID7_MINOR GENMASK(3, 0) +#define ARM_SMMU_ID7_MAJOR GENMASK(7, 4) +#define ARM_SMMU_ID7_MINOR GENMASK(3, 0) #define ARM_SMMU_GR0_sGFSR 0x48 -#define sGFSR_USF BIT(1) +#define ARM_SMMU_sGFSR_USF BIT(1) #define ARM_SMMU_GR0_sGFSYNR0 0x50 #define ARM_SMMU_GR0_sGFSYNR1 0x54 @@ -92,106 +93,132 @@ #define ARM_SMMU_GR0_sTLBGSYNC 0x70 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74 -#define sTLBGSTATUS_GSACTIVE BIT(0) +#define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0) /* Stream mapping registers */ #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) -#define SMR_VALID BIT(31) -#define SMR_MASK GENMASK(31, 16) -#define SMR_ID GENMASK(15, 0) +#define ARM_SMMU_SMR_VALID BIT(31) +#define ARM_SMMU_SMR_MASK GENMASK(31, 16) +#define ARM_SMMU_SMR_ID GENMASK(15, 0) #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) -#define S2CR_PRIVCFG GENMASK(25, 24) +#define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24) enum arm_smmu_s2cr_privcfg { S2CR_PRIVCFG_DEFAULT, S2CR_PRIVCFG_DIPAN, S2CR_PRIVCFG_UNPRIV, S2CR_PRIVCFG_PRIV, }; -#define S2CR_TYPE GENMASK(17, 16) +#define ARM_SMMU_S2CR_TYPE GENMASK(17, 16) enum arm_smmu_s2cr_type { S2CR_TYPE_TRANS, S2CR_TYPE_BYPASS, S2CR_TYPE_FAULT, }; -#define S2CR_EXIDVALID BIT(10) -#define S2CR_CBNDX GENMASK(7, 0) +#define ARM_SMMU_S2CR_EXIDVALID BIT(10) +#define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0) /* Context bank attribute registers */ #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) -#define CBAR_IRPTNDX GENMASK(31, 24) -#define CBAR_TYPE GENMASK(17, 16) +#define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24) +#define ARM_SMMU_CBAR_TYPE GENMASK(17, 16) enum arm_smmu_cbar_type { CBAR_TYPE_S2_TRANS, CBAR_TYPE_S1_TRANS_S2_BYPASS, CBAR_TYPE_S1_TRANS_S2_FAULT, CBAR_TYPE_S1_TRANS_S2_TRANS, }; -#define CBAR_S1_MEMATTR GENMASK(15, 12) -#define CBAR_S1_MEMATTR_WB 0xf -#define CBAR_S1_BPSHCFG GENMASK(9, 8) -#define CBAR_S1_BPSHCFG_NSH 3 -#define CBAR_VMID GENMASK(7, 0) +#define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12) +#define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf +#define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8) +#define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3 +#define ARM_SMMU_CBAR_VMID GENMASK(7, 0) #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) -#define CBA2R_VMID16 GENMASK(31, 16) -#define CBA2R_VA64 BIT(0) +#define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16) +#define ARM_SMMU_CBA2R_VA64 BIT(0) #define ARM_SMMU_CB_SCTLR 0x0 -#define SCTLR_S1_ASIDPNE BIT(12) -#define SCTLR_CFCFG BIT(7) -#define SCTLR_CFIE BIT(6) -#define SCTLR_CFRE BIT(5) -#define SCTLR_E BIT(4) -#define SCTLR_AFE BIT(2) -#define SCTLR_TRE BIT(1) -#define SCTLR_M BIT(0) +#define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12) +#define ARM_SMMU_SCTLR_CFCFG BIT(7) +#define ARM_SMMU_SCTLR_CFIE BIT(6) +#define ARM_SMMU_SCTLR_CFRE BIT(5) +#define ARM_SMMU_SCTLR_E BIT(4) +#define ARM_SMMU_SCTLR_AFE BIT(2) +#define ARM_SMMU_SCTLR_TRE BIT(1) +#define ARM_SMMU_SCTLR_M BIT(0) #define ARM_SMMU_CB_ACTLR 0x4 #define ARM_SMMU_CB_RESUME 0x8 -#define RESUME_TERMINATE BIT(0) +#define ARM_SMMU_RESUME_TERMINATE BIT(0) #define ARM_SMMU_CB_TCR2 0x10 -#define TCR2_SEP GENMASK(17, 15) -#define TCR2_SEP_UPSTREAM 0x7 -#define TCR2_AS BIT(4) +#define ARM_SMMU_TCR2_SEP GENMASK(17, 15) +#define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7 +#define ARM_SMMU_TCR2_AS BIT(4) +#define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0) #define ARM_SMMU_CB_TTBR0 0x20 #define ARM_SMMU_CB_TTBR1 0x28 -#define TTBRn_ASID GENMASK_ULL(63, 48) +#define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48) #define ARM_SMMU_CB_TCR 0x30 +#define ARM_SMMU_TCR_EAE BIT(31) +#define ARM_SMMU_TCR_EPD1 BIT(23) +#define ARM_SMMU_TCR_TG0 GENMASK(15, 14) +#define ARM_SMMU_TCR_SH0 GENMASK(13, 12) +#define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10) +#define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8) +#define ARM_SMMU_TCR_T0SZ GENMASK(5, 0) + +#define ARM_SMMU_VTCR_RES1 BIT(31) +#define ARM_SMMU_VTCR_PS GENMASK(18, 16) +#define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0 +#define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0 +#define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0 +#define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0 +#define ARM_SMMU_VTCR_SL0 GENMASK(7, 6) +#define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ + #define ARM_SMMU_CB_CONTEXTIDR 0x34 #define ARM_SMMU_CB_S1_MAIR0 0x38 #define ARM_SMMU_CB_S1_MAIR1 0x3c #define ARM_SMMU_CB_PAR 0x50 -#define CB_PAR_F BIT(0) +#define ARM_SMMU_CB_PAR_F BIT(0) #define ARM_SMMU_CB_FSR 0x58 -#define FSR_MULTI BIT(31) -#define FSR_SS BIT(30) -#define FSR_UUT BIT(8) -#define FSR_ASF BIT(7) -#define FSR_TLBLKF BIT(6) -#define FSR_TLBMCF BIT(5) -#define FSR_EF BIT(4) -#define FSR_PF BIT(3) -#define FSR_AFF BIT(2) -#define FSR_TF BIT(1) - -#define FSR_IGN (FSR_AFF | FSR_ASF | \ - FSR_TLBMCF | FSR_TLBLKF) -#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \ - FSR_EF | FSR_PF | FSR_TF | FSR_IGN) +#define ARM_SMMU_FSR_MULTI BIT(31) +#define ARM_SMMU_FSR_SS BIT(30) +#define ARM_SMMU_FSR_UUT BIT(8) +#define ARM_SMMU_FSR_ASF BIT(7) +#define ARM_SMMU_FSR_TLBLKF BIT(6) +#define ARM_SMMU_FSR_TLBMCF BIT(5) +#define ARM_SMMU_FSR_EF BIT(4) +#define ARM_SMMU_FSR_PF BIT(3) +#define ARM_SMMU_FSR_AFF BIT(2) +#define ARM_SMMU_FSR_TF BIT(1) + +#define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF | \ + ARM_SMMU_FSR_ASF | \ + ARM_SMMU_FSR_TLBMCF | \ + ARM_SMMU_FSR_TLBLKF) + +#define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI | \ + ARM_SMMU_FSR_SS | \ + ARM_SMMU_FSR_UUT | \ + ARM_SMMU_FSR_EF | \ + ARM_SMMU_FSR_PF | \ + ARM_SMMU_FSR_TF | \ + ARM_SMMU_FSR_IGN) #define ARM_SMMU_CB_FAR 0x60 #define ARM_SMMU_CB_FSYNR0 0x68 -#define FSYNR0_WNR BIT(4) +#define ARM_SMMU_FSYNR0_WNR BIT(4) #define ARM_SMMU_CB_S1_TLBIVA 0x600 #define ARM_SMMU_CB_S1_TLBIASID 0x610 @@ -203,7 +230,7 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_ATS1PR 0x800 #define ARM_SMMU_CB_ATSR 0x8f0 -#define ATSR_ACTIVE BIT(0) +#define ARM_SMMU_ATSR_ACTIVE BIT(0) /* Maximum number of context banks per SMMU */ @@ -297,7 +324,7 @@ struct arm_smmu_cfg { enum arm_smmu_cbar_type cbar; enum arm_smmu_context_fmt fmt; }; -#define INVALID_IRPTNDX 0xff +#define ARM_SMMU_INVALID_IRPTNDX 0xff enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, @@ -318,6 +345,33 @@ struct arm_smmu_domain { struct iommu_domain domain; }; +static inline u32 arm_smmu_lpae_tcr(struct io_pgtable_cfg *cfg) +{ + return ARM_SMMU_TCR_EPD1 | + FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) | + FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) | + FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) | + FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) | + FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz); +} + +static inline u32 arm_smmu_lpae_tcr2(struct io_pgtable_cfg *cfg) +{ + return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) | + FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM); +} + +static inline u32 arm_smmu_lpae_vtcr(struct io_pgtable_cfg *cfg) +{ + return ARM_SMMU_VTCR_RES1 | + FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) | + FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) | + FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) | + FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) | + FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) | + FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) | + FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz); +} /* Implementation details, yay! */ struct arm_smmu_impl { diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 3acfa6a25fa2..071bb42bbbc5 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -244,7 +244,7 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) || (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE && (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL && - info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) { + info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) { pr_warn("Device scope type does not match for %s\n", pci_name(info->dev)); return -EINVAL; @@ -1354,7 +1354,6 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, struct qi_desc desc; if (mask) { - WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT + mask)) - 1)); addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1; desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE; } else @@ -1371,6 +1370,47 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, qi_submit_sync(&desc, iommu); } +/* PASID-based IOTLB invalidation */ +void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, + unsigned long npages, bool ih) +{ + struct qi_desc desc = {.qw2 = 0, .qw3 = 0}; + + /* + * npages == -1 means a PASID-selective invalidation, otherwise, + * a positive value for Page-selective-within-PASID invalidation. + * 0 is not a valid input. + */ + if (WARN_ON(!npages)) { + pr_err("Invalid input npages = %ld\n", npages); + return; + } + + if (npages == -1) { + desc.qw0 = QI_EIOTLB_PASID(pasid) | + QI_EIOTLB_DID(did) | + QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | + QI_EIOTLB_TYPE; + desc.qw1 = 0; + } else { + int mask = ilog2(__roundup_pow_of_two(npages)); + unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask)); + + if (WARN_ON_ONCE(!ALIGN(addr, align))) + addr &= ~(align - 1); + + desc.qw0 = QI_EIOTLB_PASID(pasid) | + QI_EIOTLB_DID(did) | + QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | + QI_EIOTLB_TYPE; + desc.qw1 = QI_EIOTLB_ADDR(addr) | + QI_EIOTLB_IH(ih) | + QI_EIOTLB_AM(mask); + } + + qi_submit_sync(&desc, iommu); +} + /* * Disable Queued Invalidation interface. */ diff --git a/drivers/iommu/intel-iommu-debugfs.c b/drivers/iommu/intel-iommu-debugfs.c index 471f05d452e0..c1257bef553c 100644 --- a/drivers/iommu/intel-iommu-debugfs.c +++ b/drivers/iommu/intel-iommu-debugfs.c @@ -5,6 +5,7 @@ * Authors: Gayatri Kammela <gayatri.kammela@intel.com> * Sohil Mehta <sohil.mehta@intel.com> * Jacob Pan <jacob.jun.pan@linux.intel.com> + * Lu Baolu <baolu.lu@linux.intel.com> */ #include <linux/debugfs.h> @@ -283,6 +284,77 @@ static int dmar_translation_struct_show(struct seq_file *m, void *unused) } DEFINE_SHOW_ATTRIBUTE(dmar_translation_struct); +static inline unsigned long level_to_directory_size(int level) +{ + return BIT_ULL(VTD_PAGE_SHIFT + VTD_STRIDE_SHIFT * (level - 1)); +} + +static inline void +dump_page_info(struct seq_file *m, unsigned long iova, u64 *path) +{ + seq_printf(m, "0x%013lx |\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\n", + iova >> VTD_PAGE_SHIFT, path[5], path[4], + path[3], path[2], path[1]); +} + +static void pgtable_walk_level(struct seq_file *m, struct dma_pte *pde, + int level, unsigned long start, + u64 *path) +{ + int i; + + if (level > 5 || level < 1) + return; + + for (i = 0; i < BIT_ULL(VTD_STRIDE_SHIFT); + i++, pde++, start += level_to_directory_size(level)) { + if (!dma_pte_present(pde)) + continue; + + path[level] = pde->val; + if (dma_pte_superpage(pde) || level == 1) + dump_page_info(m, start, path); + else + pgtable_walk_level(m, phys_to_virt(dma_pte_addr(pde)), + level - 1, start, path); + path[level] = 0; + } +} + +static int show_device_domain_translation(struct device *dev, void *data) +{ + struct dmar_domain *domain = find_domain(dev); + struct seq_file *m = data; + u64 path[6] = { 0 }; + + if (!domain) + return 0; + + seq_printf(m, "Device %s with pasid %d @0x%llx\n", + dev_name(dev), domain->default_pasid, + (u64)virt_to_phys(domain->pgd)); + seq_puts(m, "IOVA_PFN\t\tPML5E\t\t\tPML4E\t\t\tPDPE\t\t\tPDE\t\t\tPTE\n"); + + pgtable_walk_level(m, domain->pgd, domain->agaw + 2, 0, path); + seq_putc(m, '\n'); + + return 0; +} + +static int domain_translation_struct_show(struct seq_file *m, void *unused) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&device_domain_lock, flags); + ret = bus_for_each_dev(&pci_bus_type, NULL, m, + show_device_domain_translation); + spin_unlock_irqrestore(&device_domain_lock, flags); + + return ret; +} +DEFINE_SHOW_ATTRIBUTE(domain_translation_struct); + #ifdef CONFIG_IRQ_REMAP static void ir_tbl_remap_entry_show(struct seq_file *m, struct intel_iommu *iommu) @@ -396,6 +468,9 @@ void __init intel_iommu_debugfs_init(void) &iommu_regset_fops); debugfs_create_file("dmar_translation_struct", 0444, intel_iommu_debug, NULL, &dmar_translation_struct_fops); + debugfs_create_file("domain_translation_struct", 0444, + intel_iommu_debug, NULL, + &domain_translation_struct_fops); #ifdef CONFIG_IRQ_REMAP debugfs_create_file("ir_translation_struct", 0444, intel_iommu_debug, NULL, &ir_translation_struct_fops); diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 932267f49f9a..9dc37672bf89 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -307,6 +307,20 @@ static int hw_pass_through = 1; */ #define DOMAIN_FLAG_LOSE_CHILDREN BIT(1) +/* + * When VT-d works in the scalable mode, it allows DMA translation to + * happen through either first level or second level page table. This + * bit marks that the DMA translation for the domain goes through the + * first level page table, otherwise, it goes through the second level. + */ +#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(2) + +/* + * Domain represents a virtual machine which demands iommu nested + * translation mode support. + */ +#define DOMAIN_FLAG_NESTING_MODE BIT(3) + #define for_each_domain_iommu(idx, domain) \ for (idx = 0; idx < g_num_of_iommus; idx++) \ if (domain->iommu_refcnt[idx]) @@ -355,9 +369,14 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, int dmar_disabled = 0; #else int dmar_disabled = 1; -#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/ +#endif /* CONFIG_INTEL_IOMMU_DEFAULT_ON */ +#ifdef INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON +int intel_iommu_sm = 1; +#else int intel_iommu_sm; +#endif /* INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON */ + int intel_iommu_enabled = 0; EXPORT_SYMBOL_GPL(intel_iommu_enabled); @@ -368,7 +387,6 @@ static int intel_iommu_superpage = 1; static int iommu_identity_mapping; static int intel_no_bounce; -#define IDENTMAP_ALL 1 #define IDENTMAP_GFX 2 #define IDENTMAP_AZALIA 4 @@ -377,7 +395,7 @@ EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1)) #define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2)) -static DEFINE_SPINLOCK(device_domain_lock); +DEFINE_SPINLOCK(device_domain_lock); static LIST_HEAD(device_domain_list); #define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \ @@ -552,6 +570,11 @@ static inline int domain_type_is_si(struct dmar_domain *domain) return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY; } +static inline bool domain_use_first_level(struct dmar_domain *domain) +{ + return domain->flags & DOMAIN_FLAG_USE_FIRST_LEVEL; +} + static inline int domain_pfn_supported(struct dmar_domain *domain, unsigned long pfn) { @@ -661,11 +684,12 @@ static int domain_update_iommu_snooping(struct intel_iommu *skip) return ret; } -static int domain_update_iommu_superpage(struct intel_iommu *skip) +static int domain_update_iommu_superpage(struct dmar_domain *domain, + struct intel_iommu *skip) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; - int mask = 0xf; + int mask = 0x3; if (!intel_iommu_superpage) { return 0; @@ -675,7 +699,13 @@ static int domain_update_iommu_superpage(struct intel_iommu *skip) rcu_read_lock(); for_each_active_iommu(iommu, drhd) { if (iommu != skip) { - mask &= cap_super_page_val(iommu->cap); + if (domain && domain_use_first_level(domain)) { + if (!cap_fl1gp_support(iommu->cap)) + mask = 0x1; + } else { + mask &= cap_super_page_val(iommu->cap); + } + if (!mask) break; } @@ -690,7 +720,7 @@ static void domain_update_iommu_cap(struct dmar_domain *domain) { domain_update_iommu_coherency(domain); domain->iommu_snooping = domain_update_iommu_snooping(NULL); - domain->iommu_superpage = domain_update_iommu_superpage(NULL); + domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL); } struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, @@ -774,13 +804,7 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf if (dev_is_pci(dev)) { struct pci_dev *pf_pdev; - pdev = to_pci_dev(dev); - -#ifdef CONFIG_X86 - /* VMD child devices currently cannot be handled individually */ - if (is_vmd(pdev->bus)) - return NULL; -#endif + pdev = pci_real_dma_dev(to_pci_dev(dev)); /* VFs aren't listed in scope tables; we need to look up * the PF instead to find the IOMMU. */ @@ -913,6 +937,8 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE); pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE; + if (domain_use_first_level(domain)) + pteval |= DMA_FL_PTE_XD; if (cmpxchg64(&pte->val, 0ULL, pteval)) /* Someone else set it while we were thinking; use theirs. */ free_pgtable_page(tmp_page); @@ -1483,6 +1509,20 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, spin_unlock_irqrestore(&device_domain_lock, flags); } +static void domain_flush_piotlb(struct intel_iommu *iommu, + struct dmar_domain *domain, + u64 addr, unsigned long npages, bool ih) +{ + u16 did = domain->iommu_did[iommu->seq_id]; + + if (domain->default_pasid) + qi_flush_piotlb(iommu, did, domain->default_pasid, + addr, npages, ih); + + if (!list_empty(&domain->devices)) + qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, npages, ih); +} + static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, struct dmar_domain *domain, unsigned long pfn, unsigned int pages, @@ -1496,18 +1536,23 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, if (ih) ih = 1 << 6; - /* - * Fallback to domain selective flush if no PSI support or the size is - * too big. - * PSI requires page size to be 2 ^ x, and the base address is naturally - * aligned to the size - */ - if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap)) - iommu->flush.flush_iotlb(iommu, did, 0, 0, - DMA_TLB_DSI_FLUSH); - else - iommu->flush.flush_iotlb(iommu, did, addr | ih, mask, - DMA_TLB_PSI_FLUSH); + + if (domain_use_first_level(domain)) { + domain_flush_piotlb(iommu, domain, addr, pages, ih); + } else { + /* + * Fallback to domain selective flush if no PSI support or + * the size is too big. PSI requires page size to be 2 ^ x, + * and the base address is naturally aligned to the size. + */ + if (!cap_pgsel_inv(iommu->cap) || + mask > cap_max_amask_val(iommu->cap)) + iommu->flush.flush_iotlb(iommu, did, 0, 0, + DMA_TLB_DSI_FLUSH); + else + iommu->flush.flush_iotlb(iommu, did, addr | ih, mask, + DMA_TLB_PSI_FLUSH); + } /* * In caching mode, changes of pages from non-present to present require @@ -1522,8 +1567,11 @@ static inline void __mapping_notify_one(struct intel_iommu *iommu, struct dmar_domain *domain, unsigned long pfn, unsigned int pages) { - /* It's a non-present to present mapping. Only flush if caching mode */ - if (cap_caching_mode(iommu->cap)) + /* + * It's a non-present to present mapping. Only flush if caching mode + * and second level. + */ + if (cap_caching_mode(iommu->cap) && !domain_use_first_level(domain)) iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1); else iommu_flush_write_buffer(iommu); @@ -1540,7 +1588,11 @@ static void iommu_flush_iova(struct iova_domain *iovad) struct intel_iommu *iommu = g_iommus[idx]; u16 did = domain->iommu_did[iommu->seq_id]; - iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); + if (domain_use_first_level(domain)) + domain_flush_piotlb(iommu, domain, 0, -1, 0); + else + iommu->flush.flush_iotlb(iommu, did, 0, 0, + DMA_TLB_DSI_FLUSH); if (!cap_caching_mode(iommu->cap)) iommu_flush_dev_iotlb(get_iommu_domain(iommu, did), @@ -1709,6 +1761,33 @@ static void free_dmar_iommu(struct intel_iommu *iommu) #endif } +/* + * Check and return whether first level is used by default for + * DMA translation. + */ +static bool first_level_by_default(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu; + static int first_level_support = -1; + + if (likely(first_level_support != -1)) + return first_level_support; + + first_level_support = 1; + + rcu_read_lock(); + for_each_active_iommu(iommu, drhd) { + if (!sm_supported(iommu) || !ecap_flts(iommu->ecap)) { + first_level_support = 0; + break; + } + } + rcu_read_unlock(); + + return first_level_support; +} + static struct dmar_domain *alloc_domain(int flags) { struct dmar_domain *domain; @@ -1720,6 +1799,8 @@ static struct dmar_domain *alloc_domain(int flags) memset(domain, 0, sizeof(*domain)); domain->nid = NUMA_NO_NODE; domain->flags = flags; + if (first_level_by_default()) + domain->flags |= DOMAIN_FLAG_USE_FIRST_LEVEL; domain->has_iotlb_device = false; INIT_LIST_HEAD(&domain->devices); @@ -1849,14 +1930,16 @@ static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu, { int adjust_width, agaw; unsigned long sagaw; - int err; + int ret; init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN); - err = init_iova_flush_queue(&domain->iovad, - iommu_flush_iova, iova_entry_free); - if (err) - return err; + if (!intel_iommu_strict) { + ret = init_iova_flush_queue(&domain->iovad, + iommu_flush_iova, iova_entry_free); + if (ret) + pr_info("iova flush queue initialization failed\n"); + } domain_reserve_special_ranges(domain); @@ -2229,17 +2312,20 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, unsigned long sg_res = 0; unsigned int largepage_lvl = 0; unsigned long lvl_pages = 0; + u64 attr; BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1)); if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0) return -EINVAL; - prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP; + attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP); + if (domain_use_first_level(domain)) + attr |= DMA_FL_PTE_PRESENT | DMA_FL_PTE_XD; if (!sg) { sg_res = nr_pages; - pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot; + pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr; } while (nr_pages > 0) { @@ -2251,7 +2337,7 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, sg_res = aligned_nrpages(sg->offset, sg->length); sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff; sg->dma_length = sg->length; - pteval = (sg_phys(sg) - pgoff) | prot; + pteval = (sg_phys(sg) - pgoff) | attr; phys_pfn = pteval >> VTD_PAGE_SHIFT; } @@ -2420,7 +2506,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain) spin_unlock_irqrestore(&device_domain_lock, flags); } -static struct dmar_domain *find_domain(struct device *dev) +struct dmar_domain *find_domain(struct device *dev) { struct device_domain_info *info; @@ -2428,6 +2514,9 @@ static struct dmar_domain *find_domain(struct device *dev) dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)) return NULL; + if (dev_is_pci(dev)) + dev = &pci_real_dma_dev(to_pci_dev(dev))->dev; + /* No lock here, assumes no domain exit in normal case */ info = dev->archdata.iommu; if (likely(info)) @@ -2463,6 +2552,36 @@ dmar_search_domain_by_dev_info(int segment, int bus, int devfn) return NULL; } +static int domain_setup_first_level(struct intel_iommu *iommu, + struct dmar_domain *domain, + struct device *dev, + int pasid) +{ + int flags = PASID_FLAG_SUPERVISOR_MODE; + struct dma_pte *pgd = domain->pgd; + int agaw, level; + + /* + * Skip top levels of page tables for iommu which has + * less agaw than default. Unnecessary for PT mode. + */ + for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { + pgd = phys_to_virt(dma_pte_addr(pgd)); + if (!dma_pte_present(pgd)) + return -ENOMEM; + } + + level = agaw_to_level(agaw); + if (level != 4 && level != 5) + return -EINVAL; + + flags |= (level == 5) ? PASID_FLAG_FL5LP : 0; + + return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid, + domain->iommu_did[iommu->seq_id], + flags); +} + static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, int bus, int devfn, struct device *dev, @@ -2562,6 +2681,9 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, if (hw_pass_through && domain_type_is_si(domain)) ret = intel_pasid_setup_pass_through(iommu, domain, dev, PASID_RID2PASID); + else if (domain_use_first_level(domain)) + ret = domain_setup_first_level(iommu, domain, dev, + PASID_RID2PASID); else ret = intel_pasid_setup_second_level(iommu, domain, dev, PASID_RID2PASID); @@ -2767,10 +2889,8 @@ static int __init si_domain_init(int hw) } /* - * Normally we use DMA domains for devices which have RMRRs. But we - * loose this requirement for graphic and usb devices. Identity map - * the RMRRs for graphic and USB devices so that they could use the - * si_domain. + * Identity map the RMRRs so that devices with RMRRs could also use + * the si_domain. */ for_each_rmrr_units(rmrr) { for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, @@ -2778,9 +2898,6 @@ static int __init si_domain_init(int hw) unsigned long long start = rmrr->base_address; unsigned long long end = rmrr->end_address; - if (device_is_rmrr_locked(dev)) - continue; - if (WARN_ON(end < start || end >> agaw_to_width(si_domain->agaw))) continue; @@ -2919,9 +3036,6 @@ static int device_def_domain_type(struct device *dev) if (dev_is_pci(dev)) { struct pci_dev *pdev = to_pci_dev(dev); - if (device_is_rmrr_locked(dev)) - return IOMMU_DOMAIN_DMA; - /* * Prevent any device marked as untrusted from getting * placed into the statically identity mapping domain. @@ -2959,13 +3073,9 @@ static int device_def_domain_type(struct device *dev) return IOMMU_DOMAIN_DMA; } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE) return IOMMU_DOMAIN_DMA; - } else { - if (device_has_rmrr(dev)) - return IOMMU_DOMAIN_DMA; } - return (iommu_identity_mapping & IDENTMAP_ALL) ? - IOMMU_DOMAIN_IDENTITY : 0; + return 0; } static void intel_iommu_init_qi(struct intel_iommu *iommu) @@ -3294,10 +3404,7 @@ static int __init init_dmars(void) if (!ecap_pass_through(iommu->ecap)) hw_pass_through = 0; -#ifdef CONFIG_INTEL_IOMMU_SVM - if (pasid_supported(iommu)) - intel_svm_init(iommu); -#endif + intel_svm_check(iommu); } /* @@ -3312,9 +3419,6 @@ static int __init init_dmars(void) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); } - if (iommu_default_passthrough()) - iommu_identity_mapping |= IDENTMAP_ALL; - #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA dmar_map_gfx = 0; #endif @@ -3387,8 +3491,21 @@ static unsigned long intel_alloc_iova(struct device *dev, { unsigned long iova_pfn; - /* Restrict dma_mask to the width that the iommu can handle */ - dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask); + /* + * Restrict dma_mask to the width that the iommu can handle. + * First-level translation restricts the input-address to a + * canonical address (i.e., address bits 63:N have the same + * value as address bit [N-1], where N is 48-bits with 4-level + * paging and 57-bits with 5-level paging). Hence, skip bit + * [N-1]. + */ + if (domain_use_first_level(domain)) + dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw - 1), + dma_mask); + else + dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), + dma_mask); + /* Ensure we reserve the whole size-aligned region */ nrpages = __roundup_pow_of_two(nrpages); @@ -3406,7 +3523,8 @@ static unsigned long intel_alloc_iova(struct device *dev, iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask), true); if (unlikely(!iova_pfn)) { - dev_err(dev, "Allocating %ld-page iova failed", nrpages); + dev_err_once(dev, "Allocating %ld-page iova failed\n", + nrpages); return 0; } @@ -3774,8 +3892,8 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele return 0; } - trace_map_sg(dev, iova_pfn << PAGE_SHIFT, - sg_phys(sglist), size << VTD_PAGE_SHIFT); + for_each_sg(sglist, sg, nelems, i) + trace_map_sg(dev, i + 1, nelems, sg); return nelems; } @@ -3987,6 +4105,9 @@ bounce_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, sg_dma_len(sg) = sg->length; } + for_each_sg(sglist, sg, nelems, i) + trace_bounce_map_sg(dev, i + 1, nelems, sg); + return nelems; out_unmap: @@ -4315,16 +4436,31 @@ static void __init init_iommu_pm_ops(void) static inline void init_iommu_pm_ops(void) {} #endif /* CONFIG_PM */ +static int rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr) +{ + if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) || + !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) || + rmrr->end_address <= rmrr->base_address || + arch_rmrr_sanity_check(rmrr)) + return -EINVAL; + + return 0; +} + int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg) { struct acpi_dmar_reserved_memory *rmrr; struct dmar_rmrr_unit *rmrru; - int ret; rmrr = (struct acpi_dmar_reserved_memory *)header; - ret = arch_rmrr_sanity_check(rmrr); - if (ret) - return ret; + if (rmrr_sanity_check(rmrr)) + WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND, + "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n" + "BIOS vendor: %s; Ver: %s; Product Version: %s\n", + rmrr->base_address, rmrr->end_address, + dmi_get_system_info(DMI_BIOS_VENDOR), + dmi_get_system_info(DMI_BIOS_VERSION), + dmi_get_system_info(DMI_PRODUCT_VERSION)); rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL); if (!rmrru) @@ -4470,7 +4606,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) iommu->name); return -ENXIO; } - sp = domain_update_iommu_superpage(iommu) - 1; + sp = domain_update_iommu_superpage(NULL, iommu) - 1; if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) { pr_warn("%s: Doesn't support large page.\n", iommu->name); @@ -4490,10 +4626,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) if (ret) goto out; -#ifdef CONFIG_INTEL_IOMMU_SVM - if (pasid_supported(iommu)) - intel_svm_init(iommu); -#endif + intel_svm_check(iommu); if (dmaru->ignored) { /* @@ -4898,7 +5031,7 @@ static int __init platform_optin_force_iommu(void) * map for all devices except those marked as being untrusted. */ if (dmar_disabled) - iommu_identity_mapping |= IDENTMAP_ALL; + iommu_set_default_passthrough(false); dmar_disabled = 0; no_iommu = 0; @@ -5198,6 +5331,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) { struct dmar_domain *dmar_domain; struct iommu_domain *domain; + int ret; switch (type) { case IOMMU_DOMAIN_DMA: @@ -5214,11 +5348,12 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) return NULL; } - if (type == IOMMU_DOMAIN_DMA && - init_iova_flush_queue(&dmar_domain->iovad, - iommu_flush_iova, iova_entry_free)) { - pr_warn("iova flush queue initialization failed\n"); - intel_iommu_strict = 1; + if (!intel_iommu_strict && type == IOMMU_DOMAIN_DMA) { + ret = init_iova_flush_queue(&dmar_domain->iovad, + iommu_flush_iova, + iova_entry_free); + if (ret) + pr_info("iova flush queue initialization failed\n"); } domain_update_iommu_cap(dmar_domain); @@ -5284,7 +5419,7 @@ static void auxiliary_unlink_device(struct dmar_domain *domain, domain->auxd_refcnt--; if (!domain->auxd_refcnt && domain->default_pasid > 0) - intel_pasid_free_id(domain->default_pasid); + ioasid_free(domain->default_pasid); } static int aux_domain_add_dev(struct dmar_domain *domain, @@ -5302,10 +5437,11 @@ static int aux_domain_add_dev(struct dmar_domain *domain, if (domain->default_pasid <= 0) { int pasid; - pasid = intel_pasid_alloc_id(domain, PASID_MIN, - pci_max_pasids(to_pci_dev(dev)), - GFP_KERNEL); - if (pasid <= 0) { + /* No private data needed for the default pasid */ + pasid = ioasid_alloc(NULL, PASID_MIN, + pci_max_pasids(to_pci_dev(dev)) - 1, + NULL); + if (pasid == INVALID_IOASID) { pr_err("Can't allocate default pasid\n"); return -ENODEV; } @@ -5323,8 +5459,12 @@ static int aux_domain_add_dev(struct dmar_domain *domain, goto attach_failed; /* Setup the PASID entry for mediated devices: */ - ret = intel_pasid_setup_second_level(iommu, domain, dev, - domain->default_pasid); + if (domain_use_first_level(domain)) + ret = domain_setup_first_level(iommu, domain, dev, + domain->default_pasid); + else + ret = intel_pasid_setup_second_level(iommu, domain, dev, + domain->default_pasid); if (ret) goto table_failed; spin_unlock(&iommu->lock); @@ -5341,7 +5481,7 @@ attach_failed: spin_unlock(&iommu->lock); spin_unlock_irqrestore(&device_domain_lock, flags); if (!domain->auxd_refcnt && domain->default_pasid > 0) - intel_pasid_free_id(domain->default_pasid); + ioasid_free(domain->default_pasid); return ret; } @@ -5595,6 +5735,24 @@ static inline bool iommu_pasid_support(void) return ret; } +static inline bool nested_mode_support(void) +{ + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu; + bool ret = true; + + rcu_read_lock(); + for_each_active_iommu(iommu, drhd) { + if (!sm_supported(iommu) || !ecap_nest(iommu->ecap)) { + ret = false; + break; + } + } + rcu_read_unlock(); + + return ret; +} + static bool intel_iommu_capable(enum iommu_cap cap) { if (cap == IOMMU_CAP_CACHE_COHERENCY) @@ -5752,15 +5910,6 @@ static void intel_iommu_get_resv_regions(struct device *device, list_add_tail(®->list, head); } -static void intel_iommu_put_resv_regions(struct device *dev, - struct list_head *head) -{ - struct iommu_resv_region *entry, *next; - - list_for_each_entry_safe(entry, next, head, list) - kfree(entry); -} - int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev) { struct device_domain_info *info; @@ -5987,10 +6136,42 @@ static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain, return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO; } +static int +intel_iommu_domain_set_attr(struct iommu_domain *domain, + enum iommu_attr attr, void *data) +{ + struct dmar_domain *dmar_domain = to_dmar_domain(domain); + unsigned long flags; + int ret = 0; + + if (domain->type != IOMMU_DOMAIN_UNMANAGED) + return -EINVAL; + + switch (attr) { + case DOMAIN_ATTR_NESTING: + spin_lock_irqsave(&device_domain_lock, flags); + if (nested_mode_support() && + list_empty(&dmar_domain->devices)) { + dmar_domain->flags |= DOMAIN_FLAG_NESTING_MODE; + dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL; + } else { + ret = -ENODEV; + } + spin_unlock_irqrestore(&device_domain_lock, flags); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + const struct iommu_ops intel_iommu_ops = { .capable = intel_iommu_capable, .domain_alloc = intel_iommu_domain_alloc, .domain_free = intel_iommu_domain_free, + .domain_set_attr = intel_iommu_domain_set_attr, .attach_dev = intel_iommu_attach_device, .detach_dev = intel_iommu_detach_device, .aux_attach_dev = intel_iommu_aux_attach_device, @@ -6002,7 +6183,7 @@ const struct iommu_ops intel_iommu_ops = { .add_device = intel_iommu_add_device, .remove_device = intel_iommu_remove_device, .get_resv_regions = intel_iommu_get_resv_regions, - .put_resv_regions = intel_iommu_put_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, .apply_resv_region = intel_iommu_apply_resv_region, .device_group = intel_iommu_device_group, .dev_has_feat = intel_iommu_dev_has_feat, diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c index 040a445be300..22b30f10b396 100644 --- a/drivers/iommu/intel-pasid.c +++ b/drivers/iommu/intel-pasid.c @@ -26,42 +26,6 @@ */ static DEFINE_SPINLOCK(pasid_lock); u32 intel_pasid_max_id = PASID_MAX; -static DEFINE_IDR(pasid_idr); - -int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp) -{ - int ret, min, max; - - min = max_t(int, start, PASID_MIN); - max = min_t(int, end, intel_pasid_max_id); - - WARN_ON(in_interrupt()); - idr_preload(gfp); - spin_lock(&pasid_lock); - ret = idr_alloc(&pasid_idr, ptr, min, max, GFP_ATOMIC); - spin_unlock(&pasid_lock); - idr_preload_end(); - - return ret; -} - -void intel_pasid_free_id(int pasid) -{ - spin_lock(&pasid_lock); - idr_remove(&pasid_idr, pasid); - spin_unlock(&pasid_lock); -} - -void *intel_pasid_lookup_id(int pasid) -{ - void *p; - - spin_lock(&pasid_lock); - p = idr_find(&pasid_idr, pasid); - spin_unlock(&pasid_lock); - - return p; -} /* * Per device pasid table management: @@ -465,6 +429,21 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, devtlb_invalidation_with_pasid(iommu, dev, pasid); } +static void pasid_flush_caches(struct intel_iommu *iommu, + struct pasid_entry *pte, + int pasid, u16 did) +{ + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + + if (cap_caching_mode(iommu->cap)) { + pasid_cache_invalidation_with_pasid(iommu, did, pasid); + iotlb_invalidation_with_pasid(iommu, did, pasid); + } else { + iommu_flush_write_buffer(iommu); + } +} + /* * Set up the scalable mode pasid table entry for first only * translation type. @@ -498,10 +477,15 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, pasid_set_sre(pte); } -#ifdef CONFIG_X86 - if (cpu_feature_enabled(X86_FEATURE_LA57)) - pasid_set_flpm(pte, 1); -#endif /* CONFIG_X86 */ + if (flags & PASID_FLAG_FL5LP) { + if (cap_5lp_support(iommu->cap)) { + pasid_set_flpm(pte, 1); + } else { + pr_err("No 5-level paging support for first-level\n"); + pasid_clear_entry(pte); + return -EINVAL; + } + } pasid_set_domain_id(pte, did); pasid_set_address_width(pte, iommu->agaw); @@ -510,16 +494,7 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, /* Setup Present and PASID Granular Transfer Type: */ pasid_set_translation_type(pte, 1); pasid_set_present(pte); - - if (!ecap_coherent(iommu->ecap)) - clflush_cache_range(pte, sizeof(*pte)); - - if (cap_caching_mode(iommu->cap)) { - pasid_cache_invalidation_with_pasid(iommu, did, pasid); - iotlb_invalidation_with_pasid(iommu, did, pasid); - } else { - iommu_flush_write_buffer(iommu); - } + pasid_flush_caches(iommu, pte, pasid, did); return 0; } @@ -583,16 +558,7 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, */ pasid_set_sre(pte); pasid_set_present(pte); - - if (!ecap_coherent(iommu->ecap)) - clflush_cache_range(pte, sizeof(*pte)); - - if (cap_caching_mode(iommu->cap)) { - pasid_cache_invalidation_with_pasid(iommu, did, pasid); - iotlb_invalidation_with_pasid(iommu, did, pasid); - } else { - iommu_flush_write_buffer(iommu); - } + pasid_flush_caches(iommu, pte, pasid, did); return 0; } @@ -626,16 +592,7 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, */ pasid_set_sre(pte); pasid_set_present(pte); - - if (!ecap_coherent(iommu->ecap)) - clflush_cache_range(pte, sizeof(*pte)); - - if (cap_caching_mode(iommu->cap)) { - pasid_cache_invalidation_with_pasid(iommu, did, pasid); - iotlb_invalidation_with_pasid(iommu, did, pasid); - } else { - iommu_flush_write_buffer(iommu); - } + pasid_flush_caches(iommu, pte, pasid, did); return 0; } diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index fc8cd8f17de1..92de6df24ccb 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -37,6 +37,12 @@ */ #define PASID_FLAG_SUPERVISOR_MODE BIT(0) +/* + * The PASID_FLAG_FL5LP flag Indicates using 5-level paging for first- + * level translation, otherwise, 4-level paging will be used. + */ +#define PASID_FLAG_FL5LP BIT(1) + struct pasid_dir_entry { u64 val; }; diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index dca88f9fdf29..d7f2a5358900 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -17,25 +17,13 @@ #include <linux/dmar.h> #include <linux/interrupt.h> #include <linux/mm_types.h> +#include <linux/ioasid.h> #include <asm/page.h> #include "intel-pasid.h" static irqreturn_t prq_event_thread(int irq, void *d); -int intel_svm_init(struct intel_iommu *iommu) -{ - if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && - !cap_fl1gp_support(iommu->cap)) - return -EINVAL; - - if (cpu_feature_enabled(X86_FEATURE_LA57) && - !cap_5lp_support(iommu->cap)) - return -EINVAL; - - return 0; -} - #define PRQ_ORDER 0 int intel_svm_enable_prq(struct intel_iommu *iommu) @@ -99,6 +87,33 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) return 0; } +static inline bool intel_svm_capable(struct intel_iommu *iommu) +{ + return iommu->flags & VTD_FLAG_SVM_CAPABLE; +} + +void intel_svm_check(struct intel_iommu *iommu) +{ + if (!pasid_supported(iommu)) + return; + + if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && + !cap_fl1gp_support(iommu->cap)) { + pr_err("%s SVM disabled, incompatible 1GB page capability\n", + iommu->name); + return; + } + + if (cpu_feature_enabled(X86_FEATURE_LA57) && + !cap_5lp_support(iommu->cap)) { + pr_err("%s SVM disabled, incompatible paging mode\n", + iommu->name); + return; + } + + iommu->flags |= VTD_FLAG_SVM_CAPABLE; +} + static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev, unsigned long address, unsigned long pages, int ih) { @@ -207,6 +222,10 @@ static const struct mmu_notifier_ops intel_mmuops = { static DEFINE_MUTEX(pasid_mutex); static LIST_HEAD(global_svm_list); +#define for_each_svm_dev(sdev, svm, d) \ + list_for_each_entry((sdev), &(svm)->devs, list) \ + if ((d) != (sdev)->dev) {} else + int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops) { struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); @@ -220,6 +239,9 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ if (!iommu || dmar_disabled) return -EINVAL; + if (!intel_svm_capable(iommu)) + return -ENOTSUPP; + if (dev_is_pci(dev)) { pasid_max = pci_max_pasids(to_pci_dev(dev)); if (pasid_max < 0) @@ -252,15 +274,14 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ goto out; } - list_for_each_entry(sdev, &svm->devs, list) { - if (dev == sdev->dev) { - if (sdev->ops != ops) { - ret = -EBUSY; - goto out; - } - sdev->users++; - goto success; + /* Find the matching device in svm list */ + for_each_svm_dev(sdev, svm, dev) { + if (sdev->ops != ops) { + ret = -EBUSY; + goto out; } + sdev->users++; + goto success; } break; @@ -314,16 +335,15 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ if (pasid_max > intel_pasid_max_id) pasid_max = intel_pasid_max_id; - /* Do not use PASID 0 in caching mode (virtualised IOMMU) */ - ret = intel_pasid_alloc_id(svm, - !!cap_caching_mode(iommu->cap), - pasid_max - 1, GFP_KERNEL); - if (ret < 0) { + /* Do not use PASID 0, reserved for RID to PASID */ + svm->pasid = ioasid_alloc(NULL, PASID_MIN, + pasid_max - 1, svm); + if (svm->pasid == INVALID_IOASID) { kfree(svm); kfree(sdev); + ret = -ENOSPC; goto out; } - svm->pasid = ret; svm->notifier.ops = &intel_mmuops; svm->mm = mm; svm->flags = flags; @@ -333,7 +353,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ if (mm) { ret = mmu_notifier_register(&svm->notifier, mm); if (ret) { - intel_pasid_free_id(svm->pasid); + ioasid_free(svm->pasid); kfree(svm); kfree(sdev); goto out; @@ -344,12 +364,14 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ ret = intel_pasid_setup_first_level(iommu, dev, mm ? mm->pgd : init_mm.pgd, svm->pasid, FLPT_DEFAULT_DID, - mm ? 0 : PASID_FLAG_SUPERVISOR_MODE); + (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | + (cpu_feature_enabled(X86_FEATURE_LA57) ? + PASID_FLAG_FL5LP : 0)); spin_unlock(&iommu->lock); if (ret) { if (mm) mmu_notifier_unregister(&svm->notifier, mm); - intel_pasid_free_id(svm->pasid); + ioasid_free(svm->pasid); kfree(svm); kfree(sdev); goto out; @@ -365,7 +387,9 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ ret = intel_pasid_setup_first_level(iommu, dev, mm ? mm->pgd : init_mm.pgd, svm->pasid, FLPT_DEFAULT_DID, - mm ? 0 : PASID_FLAG_SUPERVISOR_MODE); + (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | + (cpu_feature_enabled(X86_FEATURE_LA57) ? + PASID_FLAG_FL5LP : 0)); spin_unlock(&iommu->lock); if (ret) { kfree(sdev); @@ -397,44 +421,45 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) if (!iommu) goto out; - svm = intel_pasid_lookup_id(pasid); + svm = ioasid_find(NULL, pasid, NULL); if (!svm) goto out; - list_for_each_entry(sdev, &svm->devs, list) { - if (dev == sdev->dev) { - ret = 0; - sdev->users--; - if (!sdev->users) { - list_del_rcu(&sdev->list); - /* Flush the PASID cache and IOTLB for this device. - * Note that we do depend on the hardware *not* using - * the PASID any more. Just as we depend on other - * devices never using PASIDs that they have no right - * to use. We have a *shared* PASID table, because it's - * large and has to be physically contiguous. So it's - * hard to be as defensive as we might like. */ - intel_pasid_tear_down_entry(iommu, dev, svm->pasid); - intel_flush_svm_range_dev(svm, sdev, 0, -1, 0); - kfree_rcu(sdev, rcu); - - if (list_empty(&svm->devs)) { - intel_pasid_free_id(svm->pasid); - if (svm->mm) - mmu_notifier_unregister(&svm->notifier, svm->mm); - - list_del(&svm->list); - - /* We mandate that no page faults may be outstanding - * for the PASID when intel_svm_unbind_mm() is called. - * If that is not obeyed, subtle errors will happen. - * Let's make them less subtle... */ - memset(svm, 0x6b, sizeof(*svm)); - kfree(svm); - } + if (IS_ERR(svm)) { + ret = PTR_ERR(svm); + goto out; + } + + for_each_svm_dev(sdev, svm, dev) { + ret = 0; + sdev->users--; + if (!sdev->users) { + list_del_rcu(&sdev->list); + /* Flush the PASID cache and IOTLB for this device. + * Note that we do depend on the hardware *not* using + * the PASID any more. Just as we depend on other + * devices never using PASIDs that they have no right + * to use. We have a *shared* PASID table, because it's + * large and has to be physically contiguous. So it's + * hard to be as defensive as we might like. */ + intel_pasid_tear_down_entry(iommu, dev, svm->pasid); + intel_flush_svm_range_dev(svm, sdev, 0, -1, 0); + kfree_rcu(sdev, rcu); + + if (list_empty(&svm->devs)) { + ioasid_free(svm->pasid); + if (svm->mm) + mmu_notifier_unregister(&svm->notifier, svm->mm); + list_del(&svm->list); + /* We mandate that no page faults may be outstanding + * for the PASID when intel_svm_unbind_mm() is called. + * If that is not obeyed, subtle errors will happen. + * Let's make them less subtle... */ + memset(svm, 0x6b, sizeof(*svm)); + kfree(svm); } - break; } + break; } out: mutex_unlock(&pasid_mutex); @@ -454,10 +479,14 @@ int intel_svm_is_pasid_valid(struct device *dev, int pasid) if (!iommu) goto out; - svm = intel_pasid_lookup_id(pasid); + svm = ioasid_find(NULL, pasid, NULL); if (!svm) goto out; + if (IS_ERR(svm)) { + ret = PTR_ERR(svm); + goto out; + } /* init_mm is used in this case */ if (!svm->mm) ret = 1; @@ -564,13 +593,12 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (!svm || svm->pasid != req->pasid) { rcu_read_lock(); - svm = intel_pasid_lookup_id(req->pasid); + svm = ioasid_find(NULL, req->pasid, NULL); /* It *can't* go away, because the driver is not permitted * to unbind the mm while any page faults are outstanding. * So we only need RCU to protect the internal idr code. */ rcu_read_unlock(); - - if (!svm) { + if (IS_ERR_OR_NULL(svm)) { pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n", iommu->name, req->pasid, ((unsigned long long *)req)[0], ((unsigned long long *)req)[1]); @@ -654,11 +682,10 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (req->priv_data_present) memcpy(&resp.qw2, req->priv_data, sizeof(req->priv_data)); + resp.qw2 = 0; + resp.qw3 = 0; + qi_submit_sync(&resp, iommu); } - resp.qw2 = 0; - resp.qw3 = 0; - qi_submit_sync(&resp, iommu); - head = (head + sizeof(*req)) & PRQ_RING_MASK; } diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 7c3bd2c3cdca..4272fe4e17f4 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -149,8 +149,6 @@ #define ARM_V7S_TTBR_IRGN_ATTR(attr) \ ((((attr) & 0x1) << 6) | (((attr) & 0x2) >> 1)) -#define ARM_V7S_TCR_PD1 BIT(5) - #ifdef CONFIG_ZONE_DMA32 #define ARM_V7S_TABLE_GFP_DMA GFP_DMA32 #define ARM_V7S_TABLE_SLAB_FLAGS SLAB_CACHE_DMA32 @@ -798,8 +796,8 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, */ cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M; - /* TCR: T0SZ=0, disable TTBR1 */ - cfg->arm_v7s_cfg.tcr = ARM_V7S_TCR_PD1; + /* TCR: T0SZ=0, EAE=0 (if applicable) */ + cfg->arm_v7s_cfg.tcr = 0; /* * TEX remap: the indices used map to the closest equivalent types @@ -822,15 +820,13 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, /* Ensure the empty pgd is visible before any actual TTBR write */ wmb(); - /* TTBRs */ - cfg->arm_v7s_cfg.ttbr[0] = virt_to_phys(data->pgd) | - ARM_V7S_TTBR_S | ARM_V7S_TTBR_NOS | - (cfg->coherent_walk ? - (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | - ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) : - (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) | - ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC))); - cfg->arm_v7s_cfg.ttbr[1] = 0; + /* TTBR */ + cfg->arm_v7s_cfg.ttbr = virt_to_phys(data->pgd) | ARM_V7S_TTBR_S | + (cfg->coherent_walk ? (ARM_V7S_TTBR_NOS | + ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) : + (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC))); return &data->iop; out_free_data: diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index bdf47f745268..983b08477e64 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -100,40 +100,29 @@ #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) /* Register bits */ -#define ARM_32_LPAE_TCR_EAE (1 << 31) -#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) +#define ARM_LPAE_TCR_TG0_4K 0 +#define ARM_LPAE_TCR_TG0_64K 1 +#define ARM_LPAE_TCR_TG0_16K 2 -#define ARM_LPAE_TCR_EPD1 (1 << 23) +#define ARM_LPAE_TCR_TG1_16K 1 +#define ARM_LPAE_TCR_TG1_4K 2 +#define ARM_LPAE_TCR_TG1_64K 3 -#define ARM_LPAE_TCR_TG0_4K (0 << 14) -#define ARM_LPAE_TCR_TG0_64K (1 << 14) -#define ARM_LPAE_TCR_TG0_16K (2 << 14) - -#define ARM_LPAE_TCR_SH0_SHIFT 12 -#define ARM_LPAE_TCR_SH0_MASK 0x3 #define ARM_LPAE_TCR_SH_NS 0 #define ARM_LPAE_TCR_SH_OS 2 #define ARM_LPAE_TCR_SH_IS 3 -#define ARM_LPAE_TCR_ORGN0_SHIFT 10 -#define ARM_LPAE_TCR_IRGN0_SHIFT 8 -#define ARM_LPAE_TCR_RGN_MASK 0x3 #define ARM_LPAE_TCR_RGN_NC 0 #define ARM_LPAE_TCR_RGN_WBWA 1 #define ARM_LPAE_TCR_RGN_WT 2 #define ARM_LPAE_TCR_RGN_WB 3 -#define ARM_LPAE_TCR_SL0_SHIFT 6 -#define ARM_LPAE_TCR_SL0_MASK 0x3 +#define ARM_LPAE_VTCR_SL0_MASK 0x3 #define ARM_LPAE_TCR_T0SZ_SHIFT 0 -#define ARM_LPAE_TCR_SZ_MASK 0xf - -#define ARM_LPAE_TCR_PS_SHIFT 16 -#define ARM_LPAE_TCR_PS_MASK 0x7 -#define ARM_LPAE_TCR_IPS_SHIFT 32 -#define ARM_LPAE_TCR_IPS_MASK 0x7 +#define ARM_LPAE_VTCR_PS_SHIFT 16 +#define ARM_LPAE_VTCR_PS_MASK 0x7 #define ARM_LPAE_TCR_PS_32_BIT 0x0ULL #define ARM_LPAE_TCR_PS_36_BIT 0x1ULL @@ -293,17 +282,11 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, { arm_lpae_iopte pte = prot; - if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) - pte |= ARM_LPAE_PTE_NS; - if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) pte |= ARM_LPAE_PTE_TYPE_PAGE; else pte |= ARM_LPAE_PTE_TYPE_BLOCK; - if (data->iop.fmt != ARM_MALI_LPAE) - pte |= ARM_LPAE_PTE_AF; - pte |= ARM_LPAE_PTE_SH_IS; pte |= paddr_to_iopte(paddr, data); __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); @@ -460,9 +443,20 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, << ARM_LPAE_PTE_ATTRINDX_SHIFT); } + if (prot & IOMMU_CACHE) + pte |= ARM_LPAE_PTE_SH_IS; + else + pte |= ARM_LPAE_PTE_SH_OS; + if (prot & IOMMU_NOEXEC) pte |= ARM_LPAE_PTE_XN; + if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + pte |= ARM_LPAE_PTE_NS; + + if (data->iop.fmt != ARM_MALI_LPAE) + pte |= ARM_LPAE_PTE_AF; + return pte; } @@ -474,6 +468,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, arm_lpae_iopte *ptep = data->pgd; int ret, lvl = data->start_level; arm_lpae_iopte prot; + long iaext = (long)iova >> cfg->ias; /* If no access, then nothing to do */ if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) @@ -482,7 +477,9 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) return -EINVAL; - if (WARN_ON(iova >> data->iop.cfg.ias || paddr >> data->iop.cfg.oas)) + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext || paddr >> cfg->oas)) return -ERANGE; prot = arm_lpae_prot_to_pte(data, iommu_prot); @@ -648,11 +645,14 @@ static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); struct io_pgtable_cfg *cfg = &data->iop.cfg; arm_lpae_iopte *ptep = data->pgd; + long iaext = (long)iova >> cfg->ias; if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) return 0; - if (WARN_ON(iova >> data->iop.cfg.ias)) + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext)) return 0; return __arm_lpae_unmap(data, gather, iova, size, data->start_level, ptep); @@ -787,9 +787,12 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) { u64 reg; struct arm_lpae_io_pgtable *data; + typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; + bool tg1; if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | - IO_PGTABLE_QUIRK_NON_STRICT)) + IO_PGTABLE_QUIRK_NON_STRICT | + IO_PGTABLE_QUIRK_ARM_TTBR1)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -798,58 +801,55 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) /* TCR */ if (cfg->coherent_walk) { - reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + tcr->sh = ARM_LPAE_TCR_SH_IS; + tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; + tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; } else { - reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | - (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | - (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT); + tcr->sh = ARM_LPAE_TCR_SH_OS; + tcr->irgn = ARM_LPAE_TCR_RGN_NC; + tcr->orgn = ARM_LPAE_TCR_RGN_NC; } + tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: - reg |= ARM_LPAE_TCR_TG0_4K; + tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K; break; case SZ_16K: - reg |= ARM_LPAE_TCR_TG0_16K; + tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K; break; case SZ_64K: - reg |= ARM_LPAE_TCR_TG0_64K; + tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K; break; } switch (cfg->oas) { case 32: - reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT); + tcr->ips = ARM_LPAE_TCR_PS_32_BIT; break; case 36: - reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT); + tcr->ips = ARM_LPAE_TCR_PS_36_BIT; break; case 40: - reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT); + tcr->ips = ARM_LPAE_TCR_PS_40_BIT; break; case 42: - reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT); + tcr->ips = ARM_LPAE_TCR_PS_42_BIT; break; case 44: - reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT); + tcr->ips = ARM_LPAE_TCR_PS_44_BIT; break; case 48: - reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); + tcr->ips = ARM_LPAE_TCR_PS_48_BIT; break; case 52: - reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_IPS_SHIFT); + tcr->ips = ARM_LPAE_TCR_PS_52_BIT; break; default: goto out_free_data; } - reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; - - /* Disable speculative walks through TTBR1 */ - reg |= ARM_LPAE_TCR_EPD1; - cfg->arm_lpae_s1_cfg.tcr = reg; + tcr->tsz = 64ULL - cfg->ias; /* MAIRs */ reg = (ARM_LPAE_MAIR_ATTR_NC @@ -872,9 +872,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) /* Ensure the empty pgd is visible before any actual TTBR write */ wmb(); - /* TTBRs */ - cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); - cfg->arm_lpae_s1_cfg.ttbr[1] = 0; + /* TTBR */ + cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); return &data->iop; out_free_data: @@ -885,8 +884,9 @@ out_free_data: static struct io_pgtable * arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) { - u64 reg, sl; + u64 sl; struct arm_lpae_io_pgtable *data; + typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; /* The NS quirk doesn't apply at stage 2 */ if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NON_STRICT)) @@ -911,55 +911,59 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) } /* VTCR */ - reg = ARM_64_LPAE_S2_TCR_RES1 | - (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + if (cfg->coherent_walk) { + vtcr->sh = ARM_LPAE_TCR_SH_IS; + vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA; + vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA; + } else { + vtcr->sh = ARM_LPAE_TCR_SH_OS; + vtcr->irgn = ARM_LPAE_TCR_RGN_NC; + vtcr->orgn = ARM_LPAE_TCR_RGN_NC; + } sl = data->start_level; switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: - reg |= ARM_LPAE_TCR_TG0_4K; + vtcr->tg = ARM_LPAE_TCR_TG0_4K; sl++; /* SL0 format is different for 4K granule size */ break; case SZ_16K: - reg |= ARM_LPAE_TCR_TG0_16K; + vtcr->tg = ARM_LPAE_TCR_TG0_16K; break; case SZ_64K: - reg |= ARM_LPAE_TCR_TG0_64K; + vtcr->tg = ARM_LPAE_TCR_TG0_64K; break; } switch (cfg->oas) { case 32: - reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT); + vtcr->ps = ARM_LPAE_TCR_PS_32_BIT; break; case 36: - reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT); + vtcr->ps = ARM_LPAE_TCR_PS_36_BIT; break; case 40: - reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT); + vtcr->ps = ARM_LPAE_TCR_PS_40_BIT; break; case 42: - reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT); + vtcr->ps = ARM_LPAE_TCR_PS_42_BIT; break; case 44: - reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT); + vtcr->ps = ARM_LPAE_TCR_PS_44_BIT; break; case 48: - reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); + vtcr->ps = ARM_LPAE_TCR_PS_48_BIT; break; case 52: - reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_PS_SHIFT); + vtcr->ps = ARM_LPAE_TCR_PS_52_BIT; break; default: goto out_free_data; } - reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; - reg |= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT; - cfg->arm_lpae_s2_cfg.vtcr = reg; + vtcr->tsz = 64ULL - cfg->ias; + vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK; /* Allocate pgd pages */ data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), @@ -982,35 +986,21 @@ out_free_data: static struct io_pgtable * arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) { - struct io_pgtable *iop; - if (cfg->ias > 32 || cfg->oas > 40) return NULL; cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); - iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie); - if (iop) { - cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE; - cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff; - } - - return iop; + return arm_64_lpae_alloc_pgtable_s1(cfg, cookie); } static struct io_pgtable * arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) { - struct io_pgtable *iop; - if (cfg->ias > 40 || cfg->oas > 40) return NULL; cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); - iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie); - if (iop) - cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff; - - return iop; + return arm_64_lpae_alloc_pgtable_s2(cfg, cookie); } static struct io_pgtable * diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index ced53e5b72b5..94394c81468f 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -63,7 +63,7 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) if (!ops) return; - iop = container_of(ops, struct io_pgtable, ops); + iop = io_pgtable_ops_to_pgtable(ops); io_pgtable_tlb_flush_all(iop); io_pgtable_init_table[iop->fmt]->free(iop); } diff --git a/drivers/iommu/iommu-sysfs.c b/drivers/iommu/iommu-sysfs.c index e436ff813e7e..99869217fbec 100644 --- a/drivers/iommu/iommu-sysfs.c +++ b/drivers/iommu/iommu-sysfs.c @@ -87,6 +87,7 @@ error: put_device(iommu->dev); return ret; } +EXPORT_SYMBOL_GPL(iommu_device_sysfs_add); void iommu_device_sysfs_remove(struct iommu_device *iommu) { @@ -94,6 +95,8 @@ void iommu_device_sysfs_remove(struct iommu_device *iommu) device_unregister(iommu->dev); iommu->dev = NULL; } +EXPORT_SYMBOL_GPL(iommu_device_sysfs_remove); + /* * IOMMU drivers can indicate a device is managed by a given IOMMU using * this interface. A link to the device will be created in the "devices" @@ -119,6 +122,7 @@ int iommu_device_link(struct iommu_device *iommu, struct device *link) return ret; } +EXPORT_SYMBOL_GPL(iommu_device_link); void iommu_device_unlink(struct iommu_device *iommu, struct device *link) { @@ -128,3 +132,4 @@ void iommu_device_unlink(struct iommu_device *iommu, struct device *link) sysfs_remove_link(&link->kobj, "iommu"); sysfs_remove_link_from_group(&iommu->dev->kobj, "devices", dev_name(link)); } +EXPORT_SYMBOL_GPL(iommu_device_unlink); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3ead597e1c57..3e3528436e0b 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -22,6 +22,7 @@ #include <linux/bitops.h> #include <linux/property.h> #include <linux/fsl/mc.h> +#include <linux/module.h> #include <trace/events/iommu.h> static struct kset *iommu_group_kset; @@ -141,6 +142,7 @@ int iommu_device_register(struct iommu_device *iommu) spin_unlock(&iommu_device_lock); return 0; } +EXPORT_SYMBOL_GPL(iommu_device_register); void iommu_device_unregister(struct iommu_device *iommu) { @@ -148,6 +150,7 @@ void iommu_device_unregister(struct iommu_device *iommu) list_del(&iommu->list); spin_unlock(&iommu_device_lock); } +EXPORT_SYMBOL_GPL(iommu_device_unregister); static struct iommu_param *iommu_get_dev_param(struct device *dev) { @@ -183,10 +186,21 @@ int iommu_probe_device(struct device *dev) if (!iommu_get_dev_param(dev)) return -ENOMEM; + if (!try_module_get(ops->owner)) { + ret = -EINVAL; + goto err_free_dev_param; + } + ret = ops->add_device(dev); if (ret) - iommu_free_dev_param(dev); + goto err_module_put; + + return 0; +err_module_put: + module_put(ops->owner); +err_free_dev_param: + iommu_free_dev_param(dev); return ret; } @@ -197,7 +211,10 @@ void iommu_release_device(struct device *dev) if (dev->iommu_group) ops->remove_device(dev); - iommu_free_dev_param(dev); + if (dev->iommu_param) { + module_put(ops->owner); + iommu_free_dev_param(dev); + } } static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, @@ -887,6 +904,7 @@ struct iommu_group *iommu_group_ref_get(struct iommu_group *group) kobject_get(group->devices_kobj); return group; } +EXPORT_SYMBOL_GPL(iommu_group_ref_get); /** * iommu_group_put - Decrement group reference @@ -1260,6 +1278,7 @@ struct iommu_group *generic_device_group(struct device *dev) { return iommu_group_alloc(); } +EXPORT_SYMBOL_GPL(generic_device_group); /* * Use standard PCI bus topology, isolation features, and DMA alias quirks @@ -1327,6 +1346,7 @@ struct iommu_group *pci_device_group(struct device *dev) /* No shared group found, allocate new */ return iommu_group_alloc(); } +EXPORT_SYMBOL_GPL(pci_device_group); /* Get the IOMMU group for device on fsl-mc bus */ struct iommu_group *fsl_mc_device_group(struct device *dev) @@ -1339,6 +1359,7 @@ struct iommu_group *fsl_mc_device_group(struct device *dev) group = iommu_group_alloc(); return group; } +EXPORT_SYMBOL_GPL(fsl_mc_device_group); /** * iommu_group_get_for_dev - Find or create the IOMMU group for a device @@ -1407,6 +1428,7 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev) return group; } +EXPORT_SYMBOL(iommu_group_get_for_dev); struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) { @@ -1537,6 +1559,11 @@ int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) { int err; + if (ops == NULL) { + bus->iommu_ops = NULL; + return 0; + } + if (bus->iommu_ops != NULL) return -EBUSY; @@ -2230,6 +2257,25 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) ops->put_resv_regions(dev, list); } +/** + * generic_iommu_put_resv_regions - Reserved region driver helper + * @dev: device for which to free reserved regions + * @list: reserved region list for device + * + * IOMMU drivers can use this to implement their .put_resv_regions() callback + * for simple reservations. Memory allocated for each reserved region will be + * freed. If an IOMMU driver allocates additional resources per region, it is + * going to have to implement a custom callback. + */ +void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list) +{ + struct iommu_resv_region *entry, *next; + + list_for_each_entry_safe(entry, next, list, list) + kfree(entry); +} +EXPORT_SYMBOL(generic_iommu_put_resv_regions); + struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, enum iommu_resv_type type) @@ -2247,6 +2293,7 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, region->type = type; return region; } +EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); static int request_default_domain_for_dev(struct device *dev, unsigned long type) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index c7a914b9bbbc..0e6a9536eca6 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -233,7 +233,7 @@ static DEFINE_MUTEX(iova_cache_mutex); struct iova *alloc_iova_mem(void) { - return kmem_cache_zalloc(iova_cache, GFP_ATOMIC); + return kmem_cache_zalloc(iova_cache, GFP_ATOMIC | __GFP_NOWARN); } EXPORT_SYMBOL(alloc_iova_mem); diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index d02edd2751f3..ecb3f9464dd5 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -374,7 +374,7 @@ static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain) u32 tmp; /* TTBR0 */ - ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; + ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr; ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr); ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32); diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 93f14bca26ee..94a6df1bddd6 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -279,8 +279,8 @@ static void __program_context(void __iomem *base, int ctx, SET_V2PCFG(base, ctx, 0x3); SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr); - SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]); - SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]); + SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr); + SET_TTBR1(base, ctx, 0); /* Set prrr and nmrr */ SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr); diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 6fc1f5ecf91e..95945f467c03 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -367,7 +367,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, /* Update the pgtable base address register of the M4U HW */ if (!data->m4u_dom) { data->m4u_dom = dom; - writel(dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK, + writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, data->base + REG_MMU_PT_BASE_ADDR); } @@ -765,7 +765,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); if (m4u_dom) - writel(m4u_dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK, + writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR); return 0; } diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 026ad2b29dcd..20738aacac89 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -8,11 +8,12 @@ #include <linux/export.h> #include <linux/iommu.h> #include <linux/limits.h> -#include <linux/pci.h> +#include <linux/module.h> #include <linux/msi.h> #include <linux/of.h> #include <linux/of_iommu.h> #include <linux/of_pci.h> +#include <linux/pci.h> #include <linux/slab.h> #include <linux/fsl/mc.h> @@ -91,16 +92,16 @@ static int of_iommu_xlate(struct device *dev, { const struct iommu_ops *ops; struct fwnode_handle *fwnode = &iommu_spec->np->fwnode; - int err; + int ret; ops = iommu_ops_from_fwnode(fwnode); if ((ops && !ops->of_xlate) || !of_device_is_available(iommu_spec->np)) return NO_IOMMU; - err = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops); - if (err) - return err; + ret = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops); + if (ret) + return ret; /* * The otherwise-empty fwspec handily serves to indicate the specific * IOMMU device we're waiting for, which will be useful if we ever get @@ -109,7 +110,12 @@ static int of_iommu_xlate(struct device *dev, if (!ops) return driver_deferred_probe_check_state(dev); - return ops->of_xlate(dev, iommu_spec); + if (!try_module_get(ops->owner)) + return -ENODEV; + + ret = ops->of_xlate(dev, iommu_spec); + module_put(ops->owner); + return ret; } struct of_pci_iommu_alias_info { @@ -179,6 +185,7 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, .np = master_np, }; + pci_request_acs(); err = pci_for_each_dma_alias(to_pci_dev(dev), of_pci_iommu_init, &info); } else if (dev_is_fsl_mc(dev)) { @@ -196,8 +203,12 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, if (err) break; } - } + fwspec = dev_iommu_fwspec_get(dev); + if (!err && fwspec) + of_property_read_u32(master_np, "pasid-num-bits", + &fwspec->num_pasid_bits); + } /* * Two success conditions can be represented by non-negative err here: diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index 52f38292df5b..39759db4f003 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -201,7 +201,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR); - if (!(fsr & FSR_FAULT)) + if (!(fsr & ARM_SMMU_FSR_FAULT)) return IRQ_NONE; fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0); @@ -215,7 +215,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) } iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr); - iommu_writel(ctx, ARM_SMMU_CB_RESUME, RESUME_TERMINATE); + iommu_writel(ctx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE); return IRQ_HANDLED; } @@ -269,18 +269,15 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, /* TTBRs */ iommu_writeq(ctx, ARM_SMMU_CB_TTBR0, - pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] | - FIELD_PREP(TTBRn_ASID, ctx->asid)); - iommu_writeq(ctx, ARM_SMMU_CB_TTBR1, - pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] | - FIELD_PREP(TTBRn_ASID, ctx->asid)); + pgtbl_cfg.arm_lpae_s1_cfg.ttbr | + FIELD_PREP(ARM_SMMU_TTBRn_ASID, ctx->asid)); + iommu_writeq(ctx, ARM_SMMU_CB_TTBR1, 0); /* TCR */ iommu_writel(ctx, ARM_SMMU_CB_TCR2, - (pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) | - FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM)); + arm_smmu_lpae_tcr2(&pgtbl_cfg)); iommu_writel(ctx, ARM_SMMU_CB_TCR, - pgtbl_cfg.arm_lpae_s1_cfg.tcr); + arm_smmu_lpae_tcr(&pgtbl_cfg) | ARM_SMMU_TCR_EAE); /* MAIRs (stage-1 only) */ iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0, @@ -289,11 +286,13 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain, pgtbl_cfg.arm_lpae_s1_cfg.mair >> 32); /* SCTLR */ - reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | - SCTLR_M | SCTLR_S1_ASIDPNE | SCTLR_CFCFG; + reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | + ARM_SMMU_SCTLR_AFE | ARM_SMMU_SCTLR_TRE | + ARM_SMMU_SCTLR_M | ARM_SMMU_SCTLR_S1_ASIDPNE | + ARM_SMMU_SCTLR_CFCFG; if (IS_ENABLED(CONFIG_BIG_ENDIAN)) - reg |= SCTLR_E; + reg |= ARM_SMMU_SCTLR_E; iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg); diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 315c7cc4f99d..cce329d71fba 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -837,14 +837,6 @@ static void viommu_get_resv_regions(struct device *dev, struct list_head *head) iommu_dma_get_resv_regions(dev, head); } -static void viommu_put_resv_regions(struct device *dev, struct list_head *head) -{ - struct iommu_resv_region *entry, *next; - - list_for_each_entry_safe(entry, next, head, list) - kfree(entry); -} - static struct iommu_ops viommu_ops; static struct virtio_driver virtio_iommu_drv; @@ -914,7 +906,7 @@ static int viommu_add_device(struct device *dev) err_unlink_dev: iommu_device_unlink(&viommu->iommu, dev); err_free_dev: - viommu_put_resv_regions(dev, &vdev->resv_regions); + generic_iommu_put_resv_regions(dev, &vdev->resv_regions); kfree(vdev); return ret; @@ -932,7 +924,7 @@ static void viommu_remove_device(struct device *dev) iommu_group_remove_device(dev); iommu_device_unlink(&vdev->viommu->iommu, dev); - viommu_put_resv_regions(dev, &vdev->resv_regions); + generic_iommu_put_resv_regions(dev, &vdev->resv_regions); kfree(vdev); } @@ -961,7 +953,7 @@ static struct iommu_ops viommu_ops = { .remove_device = viommu_remove_device, .device_group = viommu_device_group, .get_resv_regions = viommu_get_resv_regions, - .put_resv_regions = viommu_put_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, .of_xlate = viommu_of_xlate, }; diff --git a/drivers/isdn/capi/kcapi_proc.c b/drivers/isdn/capi/kcapi_proc.c index eadbe59b3753..b5ed4ea145cb 100644 --- a/drivers/isdn/capi/kcapi_proc.c +++ b/drivers/isdn/capi/kcapi_proc.c @@ -199,8 +199,8 @@ static ssize_t empty_read(struct file *file, char __user *buf, return 0; } -static const struct file_operations empty_fops = { - .read = empty_read, +static const struct proc_ops empty_proc_ops = { + .proc_read = empty_read, }; // --------------------------------------------------------------------------- @@ -214,7 +214,7 @@ kcapi_proc_init(void) proc_create_seq("capi/contrstats", 0, NULL, &seq_contrstats_ops); proc_create_seq("capi/applications", 0, NULL, &seq_applications_ops); proc_create_seq("capi/applstats", 0, NULL, &seq_applstats_ops); - proc_create("capi/driver", 0, NULL, &empty_fops); + proc_create("capi/driver", 0, NULL, &empty_proc_ops); } void diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig index 4b68520ac251..d82f1dea3711 100644 --- a/drivers/leds/Kconfig +++ b/drivers/leds/Kconfig @@ -836,6 +836,16 @@ config LEDS_LM36274 Say Y to enable the LM36274 LED driver for TI LMU devices. This supports the LED device LM36274. +config LEDS_TPS6105X + tristate "LED support for TI TPS6105X" + depends on LEDS_CLASS + depends on TPS6105X + default y if TPS6105X + help + This driver supports TPS61050/TPS61052 LED chips. + It is a single boost converter primarily for white LEDs and + audio amplifiers. + comment "LED Triggers" source "drivers/leds/trigger/Kconfig" diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile index 2da39e896ce8..d7e1107753fb 100644 --- a/drivers/leds/Makefile +++ b/drivers/leds/Makefile @@ -85,6 +85,7 @@ obj-$(CONFIG_LEDS_LM3601X) += leds-lm3601x.o obj-$(CONFIG_LEDS_TI_LMU_COMMON) += leds-ti-lmu-common.o obj-$(CONFIG_LEDS_LM3697) += leds-lm3697.o obj-$(CONFIG_LEDS_LM36274) += leds-lm36274.o +obj-$(CONFIG_LEDS_TPS6105X) += leds-tps6105x.o # LED SPI Drivers obj-$(CONFIG_LEDS_CR0014114) += leds-cr0014114.o diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c index 438774315e6c..1fc40e8af75e 100644 --- a/drivers/leds/led-class.c +++ b/drivers/leds/led-class.c @@ -19,6 +19,7 @@ #include <linux/spinlock.h> #include <linux/timer.h> #include <uapi/linux/uleds.h> +#include <linux/of.h> #include "leds.h" static struct class *leds_class; @@ -214,6 +215,98 @@ static int led_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(leds_class_dev_pm_ops, led_suspend, led_resume); +/** + * of_led_get() - request a LED device via the LED framework + * @np: device node to get the LED device from + * @index: the index of the LED + * + * Returns the LED device parsed from the phandle specified in the "leds" + * property of a device tree node or a negative error-code on failure. + */ +struct led_classdev *of_led_get(struct device_node *np, int index) +{ + struct device *led_dev; + struct led_classdev *led_cdev; + struct device_node *led_node; + + led_node = of_parse_phandle(np, "leds", index); + if (!led_node) + return ERR_PTR(-ENOENT); + + led_dev = class_find_device_by_of_node(leds_class, led_node); + of_node_put(led_node); + + if (!led_dev) + return ERR_PTR(-EPROBE_DEFER); + + led_cdev = dev_get_drvdata(led_dev); + + if (!try_module_get(led_cdev->dev->parent->driver->owner)) + return ERR_PTR(-ENODEV); + + return led_cdev; +} +EXPORT_SYMBOL_GPL(of_led_get); + +/** + * led_put() - release a LED device + * @led_cdev: LED device + */ +void led_put(struct led_classdev *led_cdev) +{ + module_put(led_cdev->dev->parent->driver->owner); +} +EXPORT_SYMBOL_GPL(led_put); + +static void devm_led_release(struct device *dev, void *res) +{ + struct led_classdev **p = res; + + led_put(*p); +} + +/** + * devm_of_led_get - Resource-managed request of a LED device + * @dev: LED consumer + * @index: index of the LED to obtain in the consumer + * + * The device node of the device is parse to find the request LED device. + * The LED device returned from this function is automatically released + * on driver detach. + * + * @return a pointer to a LED device or ERR_PTR(errno) on failure. + */ +struct led_classdev *__must_check devm_of_led_get(struct device *dev, + int index) +{ + struct led_classdev *led; + struct led_classdev **dr; + + if (!dev) + return ERR_PTR(-EINVAL); + + /* Not using device tree? */ + if (!IS_ENABLED(CONFIG_OF) || !dev->of_node) + return ERR_PTR(-ENOTSUPP); + + led = of_led_get(dev->of_node, index); + if (IS_ERR(led)) + return led; + + dr = devres_alloc(devm_led_release, sizeof(struct led_classdev *), + GFP_KERNEL); + if (!dr) { + led_put(led); + return ERR_PTR(-ENOMEM); + } + + *dr = led; + devres_add(dev, dr); + + return led; +} +EXPORT_SYMBOL_GPL(devm_of_led_get); + static int led_classdev_next_name(const char *init_name, char *name, size_t len) { @@ -276,8 +369,10 @@ int led_classdev_register_ext(struct device *parent, mutex_unlock(&led_cdev->led_access); return PTR_ERR(led_cdev->dev); } - if (init_data && init_data->fwnode) + if (init_data && init_data->fwnode) { led_cdev->dev->fwnode = init_data->fwnode; + led_cdev->dev->of_node = to_of_node(init_data->fwnode); + } if (ret) dev_warn(parent, "Led %s renamed to %s due to name collision", diff --git a/drivers/leds/leds-bd2802.c b/drivers/leds/leds-bd2802.c index e7ec6bff2b5f..bd61a823d0ca 100644 --- a/drivers/leds/leds-bd2802.c +++ b/drivers/leds/leds-bd2802.c @@ -10,7 +10,7 @@ #include <linux/module.h> #include <linux/i2c.h> -#include <linux/gpio.h> +#include <linux/gpio/consumer.h> #include <linux/delay.h> #include <linux/leds.h> #include <linux/leds-bd2802.h> @@ -67,6 +67,7 @@ struct led_state { struct bd2802_led { struct bd2802_led_platform_data *pdata; struct i2c_client *client; + struct gpio_desc *reset; struct rw_semaphore rwsem; struct led_state led[2]; @@ -200,7 +201,7 @@ static void bd2802_update_state(struct bd2802_led *led, enum led_ids id, return; if (bd2802_is_all_off(led) && !led->adf_on) { - gpio_set_value(led->pdata->reset_gpio, 0); + gpiod_set_value(led->reset, 1); return; } @@ -226,7 +227,7 @@ static void bd2802_configure(struct bd2802_led *led) static void bd2802_reset_cancel(struct bd2802_led *led) { - gpio_set_value(led->pdata->reset_gpio, 1); + gpiod_set_value(led->reset, 0); udelay(100); bd2802_configure(led); } @@ -420,7 +421,7 @@ static void bd2802_disable_adv_conf(struct bd2802_led *led) bd2802_addr_attributes[i]); if (bd2802_is_all_off(led)) - gpio_set_value(led->pdata->reset_gpio, 0); + gpiod_set_value(led->reset, 1); led->adf_on = 0; } @@ -670,8 +671,16 @@ static int bd2802_probe(struct i2c_client *client, pdata = led->pdata = dev_get_platdata(&client->dev); i2c_set_clientdata(client, led); - /* Configure RESET GPIO (L: RESET, H: RESET cancel) */ - gpio_request_one(pdata->reset_gpio, GPIOF_OUT_INIT_HIGH, "RGB_RESETB"); + /* + * Configure RESET GPIO (L: RESET, H: RESET cancel) + * + * We request the reset GPIO as OUT_LOW which means de-asserted, + * board files specifying this GPIO line in a machine descriptor + * table should take care to specify GPIO_ACTIVE_LOW for this line. + */ + led->reset = devm_gpiod_get(&client->dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(led->reset)) + return PTR_ERR(led->reset); /* Tacss = min 0.1ms */ udelay(100); @@ -685,7 +694,7 @@ static int bd2802_probe(struct i2c_client *client, dev_info(&client->dev, "return 0x%02x\n", ret); /* To save the power, reset BD2802 after detecting */ - gpio_set_value(led->pdata->reset_gpio, 0); + gpiod_set_value(led->reset, 1); /* Default attributes */ led->wave_pattern = BD2802_PATTERN_HALF; @@ -720,7 +729,7 @@ static int bd2802_remove(struct i2c_client *client) struct bd2802_led *led = i2c_get_clientdata(client); int i; - gpio_set_value(led->pdata->reset_gpio, 0); + gpiod_set_value(led->reset, 1); bd2802_unregister_led_classdev(led); if (led->adf_on) bd2802_disable_adv_conf(led); @@ -750,7 +759,7 @@ static int bd2802_suspend(struct device *dev) struct i2c_client *client = to_i2c_client(dev); struct bd2802_led *led = i2c_get_clientdata(client); - gpio_set_value(led->pdata->reset_gpio, 0); + gpiod_set_value(led->reset, 1); return 0; } diff --git a/drivers/leds/leds-lm3532.c b/drivers/leds/leds-lm3532.c index 491268bb34a7..188a57da981a 100644 --- a/drivers/leds/leds-lm3532.c +++ b/drivers/leds/leds-lm3532.c @@ -578,6 +578,12 @@ static int lm3532_parse_node(struct lm3532_data *priv) priv->runtime_ramp_down = lm3532_get_ramp_index(ramp_time); device_for_each_child_node(priv->dev, child) { + struct led_init_data idata = { + .fwnode = child, + .default_label = ":", + .devicename = priv->client->name, + }; + led = &priv->leds[i]; ret = fwnode_property_read_u32(child, "reg", &control_bank); @@ -652,7 +658,7 @@ static int lm3532_parse_node(struct lm3532_data *priv) led->led_dev.name = led->label; led->led_dev.brightness_set_blocking = lm3532_brightness_set; - ret = devm_led_classdev_register(priv->dev, &led->led_dev); + ret = devm_led_classdev_register_ext(priv->dev, &led->led_dev, &idata); if (ret) { dev_err(&priv->client->dev, "led register err: %d\n", ret); diff --git a/drivers/leds/leds-lm3642.c b/drivers/leds/leds-lm3642.c index 480575442ed8..4232906fcb75 100644 --- a/drivers/leds/leds-lm3642.c +++ b/drivers/leds/leds-lm3642.c @@ -106,7 +106,7 @@ static int lm3642_control(struct lm3642_chip_data *chip, ret = regmap_read(chip->regmap, REG_FLAG, &chip->last_flag); if (ret < 0) { dev_err(chip->dev, "Failed to read REG_FLAG Register\n"); - goto out; + return ret; } if (chip->last_flag) @@ -146,11 +146,11 @@ static int lm3642_control(struct lm3642_chip_data *chip, break; default: - return ret; + return -EINVAL; } if (ret < 0) { dev_err(chip->dev, "Failed to write REG_I_CTRL Register\n"); - goto out; + return ret; } if (chip->tx_pin) @@ -159,13 +159,12 @@ static int lm3642_control(struct lm3642_chip_data *chip, ret = regmap_update_bits(chip->regmap, REG_ENABLE, MODE_BITS_MASK << MODE_BITS_SHIFT, opmode << MODE_BITS_SHIFT); -out: return ret; } /* torch */ -/* torch pin config for lm3642*/ +/* torch pin config for lm3642 */ static ssize_t lm3642_torch_pin_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t size) @@ -178,7 +177,7 @@ static ssize_t lm3642_torch_pin_store(struct device *dev, ret = kstrtouint(buf, 10, &state); if (ret) - goto out_strtoint; + return ret; if (state != 0) state = 0x01 << TORCH_PIN_EN_SHIFT; @@ -186,16 +185,12 @@ static ssize_t lm3642_torch_pin_store(struct device *dev, ret = regmap_update_bits(chip->regmap, REG_ENABLE, TORCH_PIN_EN_MASK << TORCH_PIN_EN_SHIFT, state); - if (ret < 0) - goto out; + if (ret < 0) { + dev_err(chip->dev, "%s:i2c access fail to register\n", __func__); + return ret; + } return size; -out: - dev_err(chip->dev, "%s:i2c access fail to register\n", __func__); - return ret; -out_strtoint: - dev_err(chip->dev, "%s: fail to change str to int\n", __func__); - return ret; } static DEVICE_ATTR(torch_pin, S_IWUSR, NULL, lm3642_torch_pin_store); @@ -229,7 +224,7 @@ static ssize_t lm3642_strobe_pin_store(struct device *dev, ret = kstrtouint(buf, 10, &state); if (ret) - goto out_strtoint; + return ret; if (state != 0) state = 0x01 << STROBE_PIN_EN_SHIFT; @@ -237,16 +232,12 @@ static ssize_t lm3642_strobe_pin_store(struct device *dev, ret = regmap_update_bits(chip->regmap, REG_ENABLE, STROBE_PIN_EN_MASK << STROBE_PIN_EN_SHIFT, state); - if (ret < 0) - goto out; + if (ret < 0) { + dev_err(chip->dev, "%s:i2c access fail to register\n", __func__); + return ret; + } return size; -out: - dev_err(chip->dev, "%s:i2c access fail to register\n", __func__); - return ret; -out_strtoint: - dev_err(chip->dev, "%s: fail to change str to int\n", __func__); - return ret; } static DEVICE_ATTR(strobe_pin, S_IWUSR, NULL, lm3642_strobe_pin_store); diff --git a/drivers/leds/leds-lm3692x.c b/drivers/leds/leds-lm3692x.c index 8b408102e138..28a51aeb28de 100644 --- a/drivers/leds/leds-lm3692x.c +++ b/drivers/leds/leds-lm3692x.c @@ -6,6 +6,7 @@ #include <linux/i2c.h> #include <linux/init.h> #include <linux/leds.h> +#include <linux/log2.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/of.h> @@ -114,6 +115,9 @@ struct lm3692x_led { struct regulator *regulator; int led_enable; int model_id; + + u8 boost_ctrl, brightness_ctrl; + bool enabled; }; static const struct reg_default lm3692x_reg_defs[] = { @@ -162,44 +166,14 @@ static int lm3692x_fault_check(struct lm3692x_led *led) return read_buf; } -static int lm3692x_brightness_set(struct led_classdev *led_cdev, - enum led_brightness brt_val) -{ - struct lm3692x_led *led = - container_of(led_cdev, struct lm3692x_led, led_dev); - int ret; - int led_brightness_lsb = (brt_val >> 5); - - mutex_lock(&led->lock); - - ret = lm3692x_fault_check(led); - if (ret) { - dev_err(&led->client->dev, "Cannot read/clear faults: %d\n", - ret); - goto out; - } - - ret = regmap_write(led->regmap, LM3692X_BRT_MSB, brt_val); - if (ret) { - dev_err(&led->client->dev, "Cannot write MSB: %d\n", ret); - goto out; - } - - ret = regmap_write(led->regmap, LM3692X_BRT_LSB, led_brightness_lsb); - if (ret) { - dev_err(&led->client->dev, "Cannot write LSB: %d\n", ret); - goto out; - } -out: - mutex_unlock(&led->lock); - return ret; -} - -static int lm3692x_init(struct lm3692x_led *led) +static int lm3692x_leds_enable(struct lm3692x_led *led) { int enable_state; int ret, reg_ret; + if (led->enabled) + return 0; + if (led->regulator) { ret = regulator_enable(led->regulator); if (ret) { @@ -249,10 +223,7 @@ static int lm3692x_init(struct lm3692x_led *led) if (ret) goto out; - ret = regmap_write(led->regmap, LM3692X_BOOST_CTRL, - LM3692X_BOOST_SW_1MHZ | - LM3692X_BOOST_SW_NO_SHIFT | - LM3692X_OCP_PROT_1_5A); + ret = regmap_write(led->regmap, LM3692X_BOOST_CTRL, led->boost_ctrl); if (ret) goto out; @@ -305,6 +276,7 @@ static int lm3692x_init(struct lm3692x_led *led) ret = regmap_update_bits(led->regmap, LM3692X_EN, LM3692X_ENABLE_MASK, enable_state | LM3692X_DEVICE_EN); + led->enabled = true; return ret; out: dev_err(&led->client->dev, "Fail writing initialization values\n"); @@ -322,10 +294,92 @@ out: return ret; } +static int lm3692x_leds_disable(struct lm3692x_led *led) +{ + int ret; + + if (!led->enabled) + return 0; + + ret = regmap_update_bits(led->regmap, LM3692X_EN, LM3692X_DEVICE_EN, 0); + if (ret) { + dev_err(&led->client->dev, "Failed to disable regulator: %d\n", + ret); + return ret; + } + + if (led->enable_gpio) + gpiod_direction_output(led->enable_gpio, 0); + + if (led->regulator) { + ret = regulator_disable(led->regulator); + if (ret) + dev_err(&led->client->dev, + "Failed to disable regulator: %d\n", ret); + } + + led->enabled = false; + return ret; +} + +static int lm3692x_brightness_set(struct led_classdev *led_cdev, + enum led_brightness brt_val) +{ + struct lm3692x_led *led = + container_of(led_cdev, struct lm3692x_led, led_dev); + int ret; + int led_brightness_lsb = (brt_val >> 5); + + mutex_lock(&led->lock); + + if (brt_val == 0) { + ret = lm3692x_leds_disable(led); + goto out; + } else { + lm3692x_leds_enable(led); + } + + ret = lm3692x_fault_check(led); + if (ret) { + dev_err(&led->client->dev, "Cannot read/clear faults: %d\n", + ret); + goto out; + } + + ret = regmap_write(led->regmap, LM3692X_BRT_MSB, brt_val); + if (ret) { + dev_err(&led->client->dev, "Cannot write MSB: %d\n", ret); + goto out; + } + + ret = regmap_write(led->regmap, LM3692X_BRT_LSB, led_brightness_lsb); + if (ret) { + dev_err(&led->client->dev, "Cannot write LSB: %d\n", ret); + goto out; + } +out: + mutex_unlock(&led->lock); + return ret; +} + +static enum led_brightness lm3692x_max_brightness(struct lm3692x_led *led, + u32 max_cur) +{ + u32 max_code; + + /* see p.12 of LM36922 data sheet for brightness formula */ + max_code = ((max_cur * 1000) - 37806) / 12195; + if (max_code > 0x7FF) + max_code = 0x7FF; + + return max_code >> 3; +} + static int lm3692x_probe_dt(struct lm3692x_led *led) { struct fwnode_handle *child = NULL; struct led_init_data init_data = {}; + u32 ovp, max_cur; int ret; led->enable_gpio = devm_gpiod_get_optional(&led->client->dev, @@ -350,6 +404,32 @@ static int lm3692x_probe_dt(struct lm3692x_led *led) led->regulator = NULL; } + led->boost_ctrl = LM3692X_BOOST_SW_1MHZ | + LM3692X_BOOST_SW_NO_SHIFT | + LM3692X_OCP_PROT_1_5A; + ret = device_property_read_u32(&led->client->dev, + "ti,ovp-microvolt", &ovp); + if (ret) { + led->boost_ctrl |= LM3692X_OVP_29V; + } else { + switch (ovp) { + case 17000000: + break; + case 21000000: + led->boost_ctrl |= LM3692X_OVP_21V; + break; + case 25000000: + led->boost_ctrl |= LM3692X_OVP_25V; + break; + case 29000000: + led->boost_ctrl |= LM3692X_OVP_29V; + break; + default: + dev_err(&led->client->dev, "Invalid OVP %d\n", ovp); + return -EINVAL; + } + } + child = device_get_next_child_node(&led->client->dev, child); if (!child) { dev_err(&led->client->dev, "No LED Child node\n"); @@ -365,6 +445,10 @@ static int lm3692x_probe_dt(struct lm3692x_led *led) return ret; } + ret = fwnode_property_read_u32(child, "led-max-microamp", &max_cur); + led->led_dev.max_brightness = ret ? LED_FULL : + lm3692x_max_brightness(led, max_cur); + init_data.fwnode = child; init_data.devicename = led->client->name; init_data.default_label = ":"; @@ -407,7 +491,7 @@ static int lm3692x_probe(struct i2c_client *client, if (ret) return ret; - ret = lm3692x_init(led); + ret = lm3692x_leds_enable(led); if (ret) return ret; @@ -419,23 +503,9 @@ static int lm3692x_remove(struct i2c_client *client) struct lm3692x_led *led = i2c_get_clientdata(client); int ret; - ret = regmap_update_bits(led->regmap, LM3692X_EN, LM3692X_DEVICE_EN, 0); - if (ret) { - dev_err(&led->client->dev, "Failed to disable regulator: %d\n", - ret); + ret = lm3692x_leds_disable(led); + if (ret) return ret; - } - - if (led->enable_gpio) - gpiod_direction_output(led->enable_gpio, 0); - - if (led->regulator) { - ret = regulator_disable(led->regulator); - if (ret) - dev_err(&led->client->dev, - "Failed to disable regulator: %d\n", ret); - } - mutex_destroy(&led->lock); return 0; diff --git a/drivers/leds/leds-pca963x.c b/drivers/leds/leds-pca963x.c index 4afc317901a8..66cdc003b8f4 100644 --- a/drivers/leds/leds-pca963x.c +++ b/drivers/leds/leds-pca963x.c @@ -40,6 +40,8 @@ #define PCA963X_LED_PWM 0x2 /* Controlled through PWM */ #define PCA963X_LED_GRP_PWM 0x3 /* Controlled through PWM/GRPPWM */ +#define PCA963X_MODE2_OUTDRV 0x04 /* Open-drain or totem pole */ +#define PCA963X_MODE2_INVRT 0x10 /* Normal or inverted direction */ #define PCA963X_MODE2_DMBLNK 0x20 /* Enable blinking */ #define PCA963X_MODE1 0x00 @@ -438,12 +440,12 @@ static int pca963x_probe(struct i2c_client *client, PCA963X_MODE2); /* Configure output: open-drain or totem pole (push-pull) */ if (pdata->outdrv == PCA963X_OPEN_DRAIN) - mode2 |= 0x01; + mode2 &= ~PCA963X_MODE2_OUTDRV; else - mode2 |= 0x05; + mode2 |= PCA963X_MODE2_OUTDRV; /* Configure direction: normal or inverted */ if (pdata->dir == PCA963X_INVERTED) - mode2 |= 0x10; + mode2 |= PCA963X_MODE2_INVRT; i2c_smbus_write_byte_data(pca963x->chip->client, PCA963X_MODE2, mode2); } diff --git a/drivers/leds/leds-tps6105x.c b/drivers/leds/leds-tps6105x.c new file mode 100644 index 000000000000..09fd88a6c8f0 --- /dev/null +++ b/drivers/leds/leds-tps6105x.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019 Sven Van Asbroeck + */ + +#include <linux/leds.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/mfd/tps6105x.h> +#include <linux/regmap.h> + +struct tps6105x_priv { + struct regmap *regmap; + struct led_classdev cdev; + struct fwnode_handle *fwnode; +}; + +static void tps6105x_handle_put(void *data) +{ + struct tps6105x_priv *priv = data; + + fwnode_handle_put(priv->fwnode); +} + +static int tps6105x_brightness_set(struct led_classdev *cdev, + enum led_brightness brightness) +{ + struct tps6105x_priv *priv = container_of(cdev, struct tps6105x_priv, + cdev); + + return regmap_update_bits(priv->regmap, TPS6105X_REG_0, + TPS6105X_REG0_TORCHC_MASK, + brightness << TPS6105X_REG0_TORCHC_SHIFT); +} + +static int tps6105x_led_probe(struct platform_device *pdev) +{ + struct tps6105x *tps6105x = dev_get_platdata(&pdev->dev); + struct tps6105x_platform_data *pdata = tps6105x->pdata; + struct led_init_data init_data = { }; + struct tps6105x_priv *priv; + int ret; + + /* This instance is not set for torch mode so bail out */ + if (pdata->mode != TPS6105X_MODE_TORCH) { + dev_info(&pdev->dev, + "chip not in torch mode, exit probe"); + return -EINVAL; + } + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + /* fwnode/devicetree is optional. NULL is allowed for priv->fwnode */ + priv->fwnode = device_get_next_child_node(pdev->dev.parent, NULL); + ret = devm_add_action_or_reset(&pdev->dev, tps6105x_handle_put, priv); + if (ret) + return ret; + priv->regmap = tps6105x->regmap; + priv->cdev.brightness_set_blocking = tps6105x_brightness_set; + priv->cdev.max_brightness = 7; + init_data.devicename = "tps6105x"; + init_data.default_label = ":torch"; + init_data.fwnode = priv->fwnode; + + ret = regmap_update_bits(tps6105x->regmap, TPS6105X_REG_0, + TPS6105X_REG0_MODE_MASK | + TPS6105X_REG0_TORCHC_MASK, + TPS6105X_REG0_MODE_TORCH << + TPS6105X_REG0_MODE_SHIFT); + if (ret) + return ret; + + return devm_led_classdev_register_ext(&pdev->dev, &priv->cdev, + &init_data); +} + +static struct platform_driver led_driver = { + .probe = tps6105x_led_probe, + .driver = { + .name = "tps6105x-leds", + }, +}; + +module_platform_driver(led_driver); + +MODULE_DESCRIPTION("TPS6105x LED driver"); +MODULE_AUTHOR("Sven Van Asbroeck <TheSven73@gmail.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/macintosh/Kconfig b/drivers/macintosh/Kconfig index 574e122ae105..cbd46c1c5bf7 100644 --- a/drivers/macintosh/Kconfig +++ b/drivers/macintosh/Kconfig @@ -178,7 +178,7 @@ config THERM_ADT746X depends on I2C && I2C_POWERMAC && PPC_PMAC && !PPC_PMAC64 help This driver provides some thermostat and fan control for the - iBook G4, and the ATI based aluminium PowerBooks, allowing slightly + iBook G4, and the ATI based aluminium PowerBooks, allowing slightly better fan behaviour by default, and some manual control. config WINDFARM @@ -214,7 +214,7 @@ config WINDFARM_PM91 select I2C_POWERMAC help This driver provides thermal control for the PowerMac9,1 - which is the recent (SMU based) single CPU desktop G5 + which is the recent (SMU based) single CPU desktop G5 config WINDFARM_PM112 tristate "Support for thermal management on PowerMac11,2" @@ -242,7 +242,7 @@ config PMAC_RACKMETER depends on PPC_PMAC help This driver provides some support to control the front panel - blue LEDs "vu-meter" of the XServer macs. + blue LEDs "vu-meter" of the XServer macs. config SENSORS_AMS tristate "Apple Motion Sensor driver" diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c index 21d532a78fa4..d38fb78a3b23 100644 --- a/drivers/macintosh/via-pmu.c +++ b/drivers/macintosh/via-pmu.c @@ -212,7 +212,7 @@ static int pmu_info_proc_show(struct seq_file *m, void *v); static int pmu_irqstats_proc_show(struct seq_file *m, void *v); static int pmu_battery_proc_show(struct seq_file *m, void *v); static void pmu_pass_intr(unsigned char *data, int len); -static const struct file_operations pmu_options_proc_fops; +static const struct proc_ops pmu_options_proc_ops; #ifdef CONFIG_ADB const struct adb_driver via_pmu_driver = { @@ -573,7 +573,7 @@ static int __init via_pmu_dev_init(void) proc_pmu_irqstats = proc_create_single("interrupts", 0, proc_pmu_root, pmu_irqstats_proc_show); proc_pmu_options = proc_create("options", 0600, proc_pmu_root, - &pmu_options_proc_fops); + &pmu_options_proc_ops); } return 0; } @@ -974,13 +974,12 @@ static ssize_t pmu_options_proc_write(struct file *file, return fcount; } -static const struct file_operations pmu_options_proc_fops = { - .owner = THIS_MODULE, - .open = pmu_options_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = pmu_options_proc_write, +static const struct proc_ops pmu_options_proc_ops = { + .proc_open = pmu_options_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = pmu_options_proc_write, }; #ifdef CONFIG_ADB diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index adf26a21fcd1..74a9849ea164 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -330,6 +330,9 @@ struct cached_dev { */ atomic_t has_dirty; +#define BCH_CACHE_READA_ALL 0 +#define BCH_CACHE_READA_META_ONLY 1 + unsigned int cache_readahead_policy; struct bch_ratelimit writeback_rate; struct delayed_work writeback_rate_update; diff --git a/drivers/md/bcache/bset.h b/drivers/md/bcache/bset.h index c71365e7c1fa..a50dcfda656f 100644 --- a/drivers/md/bcache/bset.h +++ b/drivers/md/bcache/bset.h @@ -397,7 +397,8 @@ void bch_btree_keys_stats(struct btree_keys *b, struct bset_stats *state); /* Bkey utility code */ -#define bset_bkey_last(i) bkey_idx((struct bkey *) (i)->d, (i)->keys) +#define bset_bkey_last(i) bkey_idx((struct bkey *) (i)->d, \ + (unsigned int)(i)->keys) static inline struct bkey *bset_bkey_idx(struct bset *i, unsigned int idx) { diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c index 33ddc5269e8d..6730820780b0 100644 --- a/drivers/md/bcache/journal.c +++ b/drivers/md/bcache/journal.c @@ -422,7 +422,8 @@ err: static void btree_flush_write(struct cache_set *c) { struct btree *b, *t, *btree_nodes[BTREE_FLUSH_NR]; - unsigned int i, nr, ref_nr; + unsigned int i, nr; + int ref_nr; atomic_t *fifo_front_p, *now_fifo_front_p; size_t mask; diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 73478a91a342..820d8402a1dc 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -379,13 +379,20 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) goto skip; /* - * Flag for bypass if the IO is for read-ahead or background, - * unless the read-ahead request is for metadata + * If the bio is for read-ahead or background IO, bypass it or + * not depends on the following situations, + * - If the IO is for meta data, always cache it and no bypass + * - If the IO is not meta data, check dc->cache_reada_policy, + * BCH_CACHE_READA_ALL: cache it and not bypass + * BCH_CACHE_READA_META_ONLY: not cache it and bypass + * That is, read-ahead request for metadata always get cached * (eg, for gfs2 or xfs). */ - if (bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND) && - !(bio->bi_opf & (REQ_META|REQ_PRIO))) - goto skip; + if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) { + if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) && + (dc->cache_readahead_policy != BCH_CACHE_READA_ALL)) + goto skip; + } if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) || bio_sectors(bio) & (c->sb.block_size - 1)) { diff --git a/drivers/md/bcache/stats.c b/drivers/md/bcache/stats.c index ba1c93791d8d..503aafe188dc 100644 --- a/drivers/md/bcache/stats.c +++ b/drivers/md/bcache/stats.c @@ -109,9 +109,13 @@ int bch_cache_accounting_add_kobjs(struct cache_accounting *acc, void bch_cache_accounting_clear(struct cache_accounting *acc) { - memset(&acc->total.cache_hits, - 0, - sizeof(struct cache_stats)); + acc->total.cache_hits = 0; + acc->total.cache_misses = 0; + acc->total.cache_bypass_hits = 0; + acc->total.cache_bypass_misses = 0; + acc->total.cache_readaheads = 0; + acc->total.cache_miss_collisions = 0; + acc->total.sectors_bypassed = 0; } void bch_cache_accounting_destroy(struct cache_accounting *acc) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 3dea1d5acd5c..2749daf09724 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -609,12 +609,13 @@ int bch_prio_write(struct cache *ca, bool wait) return 0; } -static void prio_read(struct cache *ca, uint64_t bucket) +static int prio_read(struct cache *ca, uint64_t bucket) { struct prio_set *p = ca->disk_buckets; struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d; struct bucket *b; unsigned int bucket_nr = 0; + int ret = -EIO; for (b = ca->buckets; b < ca->buckets + ca->sb.nbuckets; @@ -627,11 +628,15 @@ static void prio_read(struct cache *ca, uint64_t bucket) prio_io(ca, bucket, REQ_OP_READ, 0); if (p->csum != - bch_crc64(&p->magic, bucket_bytes(ca) - 8)) + bch_crc64(&p->magic, bucket_bytes(ca) - 8)) { pr_warn("bad csum reading priorities"); + goto out; + } - if (p->magic != pset_magic(&ca->sb)) + if (p->magic != pset_magic(&ca->sb)) { pr_warn("bad magic reading priorities"); + goto out; + } bucket = p->next_bucket; d = p->data; @@ -640,6 +645,10 @@ static void prio_read(struct cache *ca, uint64_t bucket) b->prio = le16_to_cpu(d->prio); b->gen = b->last_gc = d->gen; } + + ret = 0; +out: + return ret; } /* Bcache device */ @@ -1873,8 +1882,10 @@ static int run_cache_set(struct cache_set *c) j = &list_entry(journal.prev, struct journal_replay, list)->j; err = "IO error reading priorities"; - for_each_cache(ca, c, i) - prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]); + for_each_cache(ca, c, i) { + if (prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev])) + goto err; + } /* * If prio_read() fails it'll call cache_set_error and we'll diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c index 733e2ddf3c78..3470fae4eabc 100644 --- a/drivers/md/bcache/sysfs.c +++ b/drivers/md/bcache/sysfs.c @@ -27,6 +27,12 @@ static const char * const bch_cache_modes[] = { NULL }; +static const char * const bch_reada_cache_policies[] = { + "all", + "meta-only", + NULL +}; + /* Default is 0 ("auto") */ static const char * const bch_stop_on_failure_modes[] = { "auto", @@ -100,6 +106,7 @@ rw_attribute(congested_write_threshold_us); rw_attribute(sequential_cutoff); rw_attribute(data_csum); rw_attribute(cache_mode); +rw_attribute(readahead_cache_policy); rw_attribute(stop_when_cache_set_failed); rw_attribute(writeback_metadata); rw_attribute(writeback_running); @@ -168,6 +175,11 @@ SHOW(__bch_cached_dev) bch_cache_modes, BDEV_CACHE_MODE(&dc->sb)); + if (attr == &sysfs_readahead_cache_policy) + return bch_snprint_string_list(buf, PAGE_SIZE, + bch_reada_cache_policies, + dc->cache_readahead_policy); + if (attr == &sysfs_stop_when_cache_set_failed) return bch_snprint_string_list(buf, PAGE_SIZE, bch_stop_on_failure_modes, @@ -353,6 +365,15 @@ STORE(__cached_dev) } } + if (attr == &sysfs_readahead_cache_policy) { + v = __sysfs_match_string(bch_reada_cache_policies, -1, buf); + if (v < 0) + return v; + + if ((unsigned int) v != dc->cache_readahead_policy) + dc->cache_readahead_policy = v; + } + if (attr == &sysfs_stop_when_cache_set_failed) { v = __sysfs_match_string(bch_stop_on_failure_modes, -1, buf); if (v < 0) @@ -467,6 +488,7 @@ static struct attribute *bch_cached_dev_files[] = { &sysfs_data_csum, #endif &sysfs_cache_mode, + &sysfs_readahead_cache_policy, &sysfs_stop_when_cache_set_failed, &sysfs_writeback_metadata, &sysfs_writeback_running, diff --git a/drivers/md/md.c b/drivers/md/md.c index 4824d50526fa..469f551863be 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -8279,13 +8279,12 @@ static __poll_t mdstat_poll(struct file *filp, poll_table *wait) return mask; } -static const struct file_operations md_seq_fops = { - .owner = THIS_MODULE, - .open = md_seq_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release, - .poll = mdstat_poll, +static const struct proc_ops mdstat_proc_ops = { + .proc_open = md_seq_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = seq_release, + .proc_poll = mdstat_poll, }; int register_md_personality(struct md_personality *p) @@ -9454,7 +9453,7 @@ static void md_geninit(void) { pr_debug("md: sizeof(mdp_super_t) = %d\n", (int)sizeof(mdp_super_t)); - proc_create("mdstat", S_IRUGO, NULL, &md_seq_fops); + proc_create("mdstat", S_IRUGO, NULL, &mdstat_proc_ops); } static int __init md_init(void) diff --git a/drivers/media/cec/cec-core.c b/drivers/media/cec/cec-core.c index db7adffcdc76..0c52e1bb3910 100644 --- a/drivers/media/cec/cec-core.c +++ b/drivers/media/cec/cec-core.c @@ -183,24 +183,6 @@ static void cec_devnode_unregister(struct cec_adapter *adap) put_device(&devnode->dev); } -#ifdef CONFIG_CEC_NOTIFIER -static void cec_cec_notify(struct cec_adapter *adap, u16 pa) -{ - cec_s_phys_addr(adap, pa, false); -} - -void cec_register_cec_notifier(struct cec_adapter *adap, - struct cec_notifier *notifier) -{ - if (WARN_ON(!cec_is_registered(adap))) - return; - - adap->notifier = notifier; - cec_notifier_register(adap->notifier, adap, cec_cec_notify); -} -EXPORT_SYMBOL_GPL(cec_register_cec_notifier); -#endif - #ifdef CONFIG_DEBUG_FS static ssize_t cec_error_inj_write(struct file *file, const char __user *ubuf, size_t count, loff_t *ppos) @@ -416,8 +398,7 @@ void cec_unregister_adapter(struct cec_adapter *adap) #endif debugfs_remove_recursive(adap->cec_dir); #ifdef CONFIG_CEC_NOTIFIER - if (adap->notifier) - cec_notifier_unregister(adap->notifier); + cec_notifier_cec_adap_unregister(adap->notifier, adap); #endif cec_devnode_unregister(adap); } diff --git a/drivers/media/cec/cec-notifier.c b/drivers/media/cec/cec-notifier.c index 7cf42b133dbc..4a841bee5cc2 100644 --- a/drivers/media/cec/cec-notifier.c +++ b/drivers/media/cec/cec-notifier.c @@ -25,7 +25,6 @@ struct cec_notifier { struct cec_connector_info conn_info; const char *conn_name; struct cec_adapter *cec_adap; - void (*callback)(struct cec_adapter *adap, u16 pa); u16 phys_addr; }; @@ -81,13 +80,12 @@ static void cec_notifier_release(struct kref *kref) kfree(n); } -void cec_notifier_put(struct cec_notifier *n) +static void cec_notifier_put(struct cec_notifier *n) { mutex_lock(&cec_notifiers_lock); kref_put(&n->kref, cec_notifier_release); mutex_unlock(&cec_notifiers_lock); } -EXPORT_SYMBOL_GPL(cec_notifier_put); struct cec_notifier * cec_notifier_conn_register(struct device *hdmi_dev, const char *conn_name, @@ -162,7 +160,6 @@ void cec_notifier_cec_adap_unregister(struct cec_notifier *n, mutex_lock(&n->lock); adap->notifier = NULL; n->cec_adap = NULL; - n->callback = NULL; mutex_unlock(&n->lock); cec_notifier_put(n); } @@ -175,9 +172,7 @@ void cec_notifier_set_phys_addr(struct cec_notifier *n, u16 pa) mutex_lock(&n->lock); n->phys_addr = pa; - if (n->callback) - n->callback(n->cec_adap, n->phys_addr); - else if (n->cec_adap) + if (n->cec_adap) cec_s_phys_addr(n->cec_adap, n->phys_addr, false); mutex_unlock(&n->lock); } @@ -198,34 +193,6 @@ void cec_notifier_set_phys_addr_from_edid(struct cec_notifier *n, } EXPORT_SYMBOL_GPL(cec_notifier_set_phys_addr_from_edid); -void cec_notifier_register(struct cec_notifier *n, - struct cec_adapter *adap, - void (*callback)(struct cec_adapter *adap, u16 pa)) -{ - kref_get(&n->kref); - mutex_lock(&n->lock); - n->cec_adap = adap; - n->callback = callback; - n->callback(adap, n->phys_addr); - mutex_unlock(&n->lock); -} -EXPORT_SYMBOL_GPL(cec_notifier_register); - -void cec_notifier_unregister(struct cec_notifier *n) -{ - /* Do nothing unless cec_notifier_register was called first */ - if (!n->callback) - return; - - mutex_lock(&n->lock); - n->callback = NULL; - n->cec_adap->notifier = NULL; - n->cec_adap = NULL; - mutex_unlock(&n->lock); - cec_notifier_put(n); -} -EXPORT_SYMBOL_GPL(cec_notifier_unregister); - struct device *cec_notifier_parse_hdmi_phandle(struct device *dev) { struct platform_device *hdmi_pdev; diff --git a/drivers/media/cec/cec-priv.h b/drivers/media/cec/cec-priv.h index 7bdf855aaecd..9bbd05053d42 100644 --- a/drivers/media/cec/cec-priv.h +++ b/drivers/media/cec/cec-priv.h @@ -9,7 +9,7 @@ #define _CEC_PRIV_H #include <linux/cec-funcs.h> -#include <media/cec.h> +#include <media/cec-notifier.h> #define dprintk(lvl, fmt, arg...) \ do { \ diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/media/common/saa7146/saa7146_video.c index d16122039b0c..ccd15b4d4920 100644 --- a/drivers/media/common/saa7146/saa7146_video.c +++ b/drivers/media/common/saa7146/saa7146_video.c @@ -345,7 +345,8 @@ static int video_begin(struct saa7146_fh *fh) fmt = saa7146_format_by_fourcc(dev, vv->video_fmt.pixelformat); /* we need to have a valid format set here */ - BUG_ON(NULL == fmt); + if (!fmt) + return -EINVAL; if (0 != (fmt->flags & FORMAT_IS_PLANAR)) { resource = RESOURCE_DMA1_HPS|RESOURCE_DMA2_CLP|RESOURCE_DMA3_BRS; @@ -398,7 +399,8 @@ static int video_end(struct saa7146_fh *fh, struct file *file) fmt = saa7146_format_by_fourcc(dev, vv->video_fmt.pixelformat); /* we need to have a valid format set here */ - BUG_ON(NULL == fmt); + if (!fmt) + return -EINVAL; if (0 != (fmt->flags & FORMAT_IS_PLANAR)) { resource = RESOURCE_DMA1_HPS|RESOURCE_DMA2_CLP|RESOURCE_DMA3_BRS; diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c index e652f4318284..eb5d5db96552 100644 --- a/drivers/media/common/videobuf2/videobuf2-v4l2.c +++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c @@ -146,7 +146,7 @@ static void __copy_timestamp(struct vb2_buffer *vb, const void *pb) * and the timecode field and flag if needed. */ if (q->copy_timestamp) - vb->timestamp = v4l2_timeval_to_ns(&b->timestamp); + vb->timestamp = v4l2_buffer_get_timestamp(b); vbuf->flags |= b->flags & V4L2_BUF_FLAG_TIMECODE; if (b->flags & V4L2_BUF_FLAG_TIMECODE) vbuf->timecode = b->timecode; @@ -482,7 +482,7 @@ static void __fill_v4l2_buffer(struct vb2_buffer *vb, void *pb) b->flags = vbuf->flags; b->field = vbuf->field; - b->timestamp = ns_to_timeval(vb->timestamp); + v4l2_buffer_set_timestamp(b, vb->timestamp); b->timecode = vbuf->timecode; b->sequence = vbuf->sequence; b->reserved2 = 0; diff --git a/drivers/media/dvb-core/dvb_demux.c b/drivers/media/dvb-core/dvb_demux.c index 39a2c6ccf31d..5fde1d38b3e3 100644 --- a/drivers/media/dvb-core/dvb_demux.c +++ b/drivers/media/dvb-core/dvb_demux.c @@ -971,6 +971,7 @@ static int dmx_section_feed_start_filtering(struct dmx_section_feed *feed) dvbdmxfeed->feed.sec.secbuf = dvbdmxfeed->feed.sec.secbuf_base; dvbdmxfeed->feed.sec.secbufp = 0; dvbdmxfeed->feed.sec.seclen = 0; + dvbdmxfeed->pusi_seen = false; if (!dvbdmx->start_feed) { mutex_unlock(&dvbdmx->mutex); diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c index 917fe034af37..80b6a71aa33e 100644 --- a/drivers/media/dvb-core/dvbdev.c +++ b/drivers/media/dvb-core/dvbdev.c @@ -983,8 +983,8 @@ struct i2c_client *dvb_module_probe(const char *module_name, board_info->addr = addr; board_info->platform_data = platform_data; request_module(module_name); - client = i2c_new_device(adap, board_info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(adap, board_info); + if (!i2c_client_has_driver(client)) { kfree(board_info); return NULL; } diff --git a/drivers/media/dvb-frontends/as102_fe.c b/drivers/media/dvb-frontends/as102_fe.c index 496ebb8176c0..bc72d954dc1f 100644 --- a/drivers/media/dvb-frontends/as102_fe.c +++ b/drivers/media/dvb-frontends/as102_fe.c @@ -290,7 +290,8 @@ static int as102_fe_get_frontend(struct dvb_frontend *fe, } static int as102_fe_get_tune_settings(struct dvb_frontend *fe, - struct dvb_frontend_tune_settings *settings) { + struct dvb_frontend_tune_settings *settings) +{ settings->min_delay_ms = 1000; diff --git a/drivers/media/dvb-frontends/au8522_decoder.c b/drivers/media/dvb-frontends/au8522_decoder.c index c1717dde874b..8cdca051e51b 100644 --- a/drivers/media/dvb-frontends/au8522_decoder.c +++ b/drivers/media/dvb-frontends/au8522_decoder.c @@ -562,7 +562,7 @@ static int au8522_s_video_routing(struct v4l2_subdev *sd, { struct au8522_state *state = to_state(sd); - switch(input) { + switch (input) { case AU8522_COMPOSITE_CH1: case AU8522_SVIDEO_CH13: case AU8522_COMPOSITE_CH4_SIF: diff --git a/drivers/media/dvb-frontends/cxd2820r_core.c b/drivers/media/dvb-frontends/cxd2820r_core.c index d137199e13e6..b1618339eec0 100644 --- a/drivers/media/dvb-frontends/cxd2820r_core.c +++ b/drivers/media/dvb-frontends/cxd2820r_core.c @@ -530,8 +530,8 @@ struct dvb_frontend *cxd2820r_attach(const struct cxd2820r_config *config, strscpy(board_info.type, "cxd2820r", I2C_NAME_SIZE); board_info.addr = config->i2c_address; board_info.platform_data = &pdata; - client = i2c_new_device(adapter, &board_info); - if (!client || !client->dev.driver) + client = i2c_new_client_device(adapter, &board_info); + if (!i2c_client_has_driver(client)) return NULL; return pdata.get_dvb_frontend(client); diff --git a/drivers/media/dvb-frontends/dib0070.c b/drivers/media/dvb-frontends/dib0070.c index 3b26f61785d8..cafb41dba861 100644 --- a/drivers/media/dvb-frontends/dib0070.c +++ b/drivers/media/dvb-frontends/dib0070.c @@ -189,7 +189,8 @@ static int dib0070_captrim(struct dib0070_state *state, enum frontend_tune_state adc = dib0070_read_reg(state, 0x19); - dprintk("CAPTRIM=%hd; ADC = %hd (ADC) & %dmV\n", state->captrim, adc, (u32) adc*(u32)1800/(u32)1024); + dprintk("CAPTRIM=%d; ADC = %hd (ADC) & %dmV\n", state->captrim, + adc, (u32)adc * (u32)1800 / (u32)1024); if (adc >= 400) { adc -= 400; @@ -200,7 +201,8 @@ static int dib0070_captrim(struct dib0070_state *state, enum frontend_tune_state } if (adc < state->adc_diff) { - dprintk("CAPTRIM=%hd is closer to target (%hd/%hd)\n", state->captrim, adc, state->adc_diff); + dprintk("CAPTRIM=%d is closer to target (%hd/%hd)\n", + state->captrim, adc, state->adc_diff); state->adc_diff = adc; state->fcaptrim = state->captrim; } @@ -364,7 +366,7 @@ static int dib0070_tune_digital(struct dvb_frontend *fe) } if (*tune_state == CT_TUNER_START) { - dprintk("Tuning for Band: %hd (%d kHz)\n", band, freq); + dprintk("Tuning for Band: %d (%d kHz)\n", band, freq); if (state->current_rf != freq) { u8 REFDIV; u32 FBDiv, Rest, FREF, VCOF_kHz; @@ -442,12 +444,17 @@ static int dib0070_tune_digital(struct dvb_frontend *fe) dib0070_write_reg(state, 0x20, 0x0040 | 0x0020 | 0x0010 | 0x0008 | 0x0002 | 0x0001 | state->current_tune_table_index->tuner_enable); - dprintk("REFDIV: %hd, FREF: %d\n", REFDIV, FREF); + dprintk("REFDIV: %u, FREF: %d\n", REFDIV, FREF); dprintk("FBDIV: %d, Rest: %d\n", FBDiv, Rest); - dprintk("Num: %hd, Den: %hd, SD: %hd\n", (u16) Rest, Den, (state->lo4 >> 12) & 0x1); - dprintk("HFDIV code: %hd\n", state->current_tune_table_index->hfdiv); - dprintk("VCO = %hd\n", state->current_tune_table_index->vco_band); - dprintk("VCOF: ((%hd*%d) << 1))\n", state->current_tune_table_index->vco_multi, freq); + dprintk("Num: %u, Den: %u, SD: %d\n", (u16)Rest, Den, + (state->lo4 >> 12) & 0x1); + dprintk("HFDIV code: %u\n", + state->current_tune_table_index->hfdiv); + dprintk("VCO = %u\n", + state->current_tune_table_index->vco_band); + dprintk("VCOF: ((%u*%d) << 1))\n", + state->current_tune_table_index->vco_multi, + freq); *tune_state = CT_TUNER_STEP_0; } else { /* we are already tuned to this frequency - the configuration is correct */ diff --git a/drivers/media/dvb-frontends/dib0090.c b/drivers/media/dvb-frontends/dib0090.c index d13d2e81f8c9..bc374750529b 100644 --- a/drivers/media/dvb-frontends/dib0090.c +++ b/drivers/media/dvb-frontends/dib0090.c @@ -1748,7 +1748,8 @@ static int dib0090_dc_offset_calibration(struct dib0090_state *state, enum front } dib0090_set_trim(state); - dprintk("BB Offset Cal, BBreg=%hd,Offset=%hd,Value Set=%hd\n", state->dc->addr, state->adc_diff, state->step); + dprintk("BB Offset Cal, BBreg=%u,Offset=%d,Value Set=%d\n", + state->dc->addr, state->adc_diff, state->step); state->dc++; if (state->dc->addr == 0) /* done */ diff --git a/drivers/media/dvb-frontends/dib7000m.c b/drivers/media/dvb-frontends/dib7000m.c index e211830c9c99..97ce97789c9e 100644 --- a/drivers/media/dvb-frontends/dib7000m.c +++ b/drivers/media/dvb-frontends/dib7000m.c @@ -808,7 +808,7 @@ static int dib7000m_agc_startup(struct dvb_frontend *demod) dib7000m_restart_agc(state); - dprintk("SPLIT %p: %hd\n", demod, agc_split); + dprintk("SPLIT %p: %u\n", demod, agc_split); (*agc_state)++; ret = 5; diff --git a/drivers/media/dvb-frontends/dib7000p.c b/drivers/media/dvb-frontends/dib7000p.c index 0d22c700016d..0a7790c4bad3 100644 --- a/drivers/media/dvb-frontends/dib7000p.c +++ b/drivers/media/dvb-frontends/dib7000p.c @@ -915,7 +915,7 @@ static int dib7000p_agc_startup(struct dvb_frontend *demod) dib7000p_restart_agc(state); - dprintk("SPLIT %p: %hd\n", demod, agc_split); + dprintk("SPLIT %p: %u\n", demod, agc_split); (*agc_state)++; ret = 5; diff --git a/drivers/media/dvb-frontends/dvb_dummy_fe.c b/drivers/media/dvb-frontends/dvb_dummy_fe.c index 4db679cb70ad..9ff1ebaa5e04 100644 --- a/drivers/media/dvb-frontends/dvb_dummy_fe.c +++ b/drivers/media/dvb-frontends/dvb_dummy_fe.c @@ -31,25 +31,26 @@ static int dvb_dummy_fe_read_status(struct dvb_frontend *fe, return 0; } -static int dvb_dummy_fe_read_ber(struct dvb_frontend* fe, u32* ber) +static int dvb_dummy_fe_read_ber(struct dvb_frontend *fe, u32 *ber) { *ber = 0; return 0; } -static int dvb_dummy_fe_read_signal_strength(struct dvb_frontend* fe, u16* strength) +static int dvb_dummy_fe_read_signal_strength(struct dvb_frontend *fe, + u16 *strength) { *strength = 0; return 0; } -static int dvb_dummy_fe_read_snr(struct dvb_frontend* fe, u16* snr) +static int dvb_dummy_fe_read_snr(struct dvb_frontend *fe, u16 *snr) { *snr = 0; return 0; } -static int dvb_dummy_fe_read_ucblocks(struct dvb_frontend* fe, u32* ucblocks) +static int dvb_dummy_fe_read_ucblocks(struct dvb_frontend *fe, u32 *ucblocks) { *ucblocks = 0; return 0; @@ -77,12 +78,12 @@ static int dvb_dummy_fe_set_frontend(struct dvb_frontend *fe) return 0; } -static int dvb_dummy_fe_sleep(struct dvb_frontend* fe) +static int dvb_dummy_fe_sleep(struct dvb_frontend *fe) { return 0; } -static int dvb_dummy_fe_init(struct dvb_frontend* fe) +static int dvb_dummy_fe_init(struct dvb_frontend *fe) { return 0; } @@ -99,17 +100,18 @@ static int dvb_dummy_fe_set_voltage(struct dvb_frontend *fe, return 0; } -static void dvb_dummy_fe_release(struct dvb_frontend* fe) +static void dvb_dummy_fe_release(struct dvb_frontend *fe) { - struct dvb_dummy_fe_state* state = fe->demodulator_priv; + struct dvb_dummy_fe_state *state = fe->demodulator_priv; + kfree(state); } static const struct dvb_frontend_ops dvb_dummy_fe_ofdm_ops; -struct dvb_frontend* dvb_dummy_fe_ofdm_attach(void) +struct dvb_frontend *dvb_dummy_fe_ofdm_attach(void) { - struct dvb_dummy_fe_state* state = NULL; + struct dvb_dummy_fe_state *state = NULL; /* allocate memory for the internal state */ state = kzalloc(sizeof(struct dvb_dummy_fe_state), GFP_KERNEL); @@ -117,16 +119,20 @@ struct dvb_frontend* dvb_dummy_fe_ofdm_attach(void) return NULL; /* create dvb_frontend */ - memcpy(&state->frontend.ops, &dvb_dummy_fe_ofdm_ops, sizeof(struct dvb_frontend_ops)); + memcpy(&state->frontend.ops, + &dvb_dummy_fe_ofdm_ops, + sizeof(struct dvb_frontend_ops)); + state->frontend.demodulator_priv = state; return &state->frontend; } +EXPORT_SYMBOL(dvb_dummy_fe_ofdm_attach); static const struct dvb_frontend_ops dvb_dummy_fe_qpsk_ops; struct dvb_frontend *dvb_dummy_fe_qpsk_attach(void) { - struct dvb_dummy_fe_state* state = NULL; + struct dvb_dummy_fe_state *state = NULL; /* allocate memory for the internal state */ state = kzalloc(sizeof(struct dvb_dummy_fe_state), GFP_KERNEL); @@ -134,16 +140,20 @@ struct dvb_frontend *dvb_dummy_fe_qpsk_attach(void) return NULL; /* create dvb_frontend */ - memcpy(&state->frontend.ops, &dvb_dummy_fe_qpsk_ops, sizeof(struct dvb_frontend_ops)); + memcpy(&state->frontend.ops, + &dvb_dummy_fe_qpsk_ops, + sizeof(struct dvb_frontend_ops)); + state->frontend.demodulator_priv = state; return &state->frontend; } +EXPORT_SYMBOL(dvb_dummy_fe_qpsk_attach); static const struct dvb_frontend_ops dvb_dummy_fe_qam_ops; struct dvb_frontend *dvb_dummy_fe_qam_attach(void) { - struct dvb_dummy_fe_state* state = NULL; + struct dvb_dummy_fe_state *state = NULL; /* allocate memory for the internal state */ state = kzalloc(sizeof(struct dvb_dummy_fe_state), GFP_KERNEL); @@ -151,10 +161,14 @@ struct dvb_frontend *dvb_dummy_fe_qam_attach(void) return NULL; /* create dvb_frontend */ - memcpy(&state->frontend.ops, &dvb_dummy_fe_qam_ops, sizeof(struct dvb_frontend_ops)); + memcpy(&state->frontend.ops, + &dvb_dummy_fe_qam_ops, + sizeof(struct dvb_frontend_ops)); + state->frontend.demodulator_priv = state; return &state->frontend; } +EXPORT_SYMBOL(dvb_dummy_fe_qam_attach); static const struct dvb_frontend_ops dvb_dummy_fe_ofdm_ops = { .delsys = { SYS_DVBT }, @@ -163,13 +177,21 @@ static const struct dvb_frontend_ops dvb_dummy_fe_ofdm_ops = { .frequency_min_hz = 0, .frequency_max_hz = 863250 * kHz, .frequency_stepsize_hz = 62500, - .caps = FE_CAN_FEC_1_2 | FE_CAN_FEC_2_3 | FE_CAN_FEC_3_4 | - FE_CAN_FEC_4_5 | FE_CAN_FEC_5_6 | FE_CAN_FEC_6_7 | - FE_CAN_FEC_7_8 | FE_CAN_FEC_8_9 | FE_CAN_FEC_AUTO | - FE_CAN_QAM_16 | FE_CAN_QAM_64 | FE_CAN_QAM_AUTO | - FE_CAN_TRANSMISSION_MODE_AUTO | - FE_CAN_GUARD_INTERVAL_AUTO | - FE_CAN_HIERARCHY_AUTO, + .caps = FE_CAN_FEC_1_2 | + FE_CAN_FEC_2_3 | + FE_CAN_FEC_3_4 | + FE_CAN_FEC_4_5 | + FE_CAN_FEC_5_6 | + FE_CAN_FEC_6_7 | + FE_CAN_FEC_7_8 | + FE_CAN_FEC_8_9 | + FE_CAN_FEC_AUTO | + FE_CAN_QAM_16 | + FE_CAN_QAM_64 | + FE_CAN_QAM_AUTO | + FE_CAN_TRANSMISSION_MODE_AUTO | + FE_CAN_GUARD_INTERVAL_AUTO | + FE_CAN_HIERARCHY_AUTO, }, .release = dvb_dummy_fe_release, @@ -194,11 +216,16 @@ static const struct dvb_frontend_ops dvb_dummy_fe_qam_ops = { .frequency_min_hz = 51 * MHz, .frequency_max_hz = 858 * MHz, .frequency_stepsize_hz = 62500, - .symbol_rate_min = (57840000 / 2) / 64, /* SACLK/64 == (XIN/2)/64 */ + /* symbol_rate_min: SACLK/64 == (XIN/2)/64 */ + .symbol_rate_min = (57840000 / 2) / 64, .symbol_rate_max = (57840000 / 2) / 4, /* SACLK/4 */ - .caps = FE_CAN_QAM_16 | FE_CAN_QAM_32 | FE_CAN_QAM_64 | - FE_CAN_QAM_128 | FE_CAN_QAM_256 | - FE_CAN_FEC_AUTO | FE_CAN_INVERSION_AUTO + .caps = FE_CAN_QAM_16 | + FE_CAN_QAM_32 | + FE_CAN_QAM_64 | + FE_CAN_QAM_128 | + FE_CAN_QAM_256 | + FE_CAN_FEC_AUTO | + FE_CAN_INVERSION_AUTO }, .release = dvb_dummy_fe_release, @@ -227,8 +254,12 @@ static const struct dvb_frontend_ops dvb_dummy_fe_qpsk_ops = { .symbol_rate_min = 1000000, .symbol_rate_max = 45000000, .caps = FE_CAN_INVERSION_AUTO | - FE_CAN_FEC_1_2 | FE_CAN_FEC_2_3 | FE_CAN_FEC_3_4 | - FE_CAN_FEC_5_6 | FE_CAN_FEC_7_8 | FE_CAN_FEC_AUTO | + FE_CAN_FEC_1_2 | + FE_CAN_FEC_2_3 | + FE_CAN_FEC_3_4 | + FE_CAN_FEC_5_6 | + FE_CAN_FEC_7_8 | + FE_CAN_FEC_AUTO | FE_CAN_QPSK }, @@ -253,7 +284,3 @@ static const struct dvb_frontend_ops dvb_dummy_fe_qpsk_ops = { MODULE_DESCRIPTION("DVB DUMMY Frontend"); MODULE_AUTHOR("Emard"); MODULE_LICENSE("GPL"); - -EXPORT_SYMBOL(dvb_dummy_fe_ofdm_attach); -EXPORT_SYMBOL(dvb_dummy_fe_qam_attach); -EXPORT_SYMBOL(dvb_dummy_fe_qpsk_attach); diff --git a/drivers/media/dvb-frontends/dvb_dummy_fe.h b/drivers/media/dvb-frontends/dvb_dummy_fe.h index 526fabd7751f..463abf5ebd56 100644 --- a/drivers/media/dvb-frontends/dvb_dummy_fe.h +++ b/drivers/media/dvb-frontends/dvb_dummy_fe.h @@ -12,23 +12,23 @@ #include <media/dvb_frontend.h> #if IS_REACHABLE(CONFIG_DVB_DUMMY_FE) -extern struct dvb_frontend* dvb_dummy_fe_ofdm_attach(void); -extern struct dvb_frontend* dvb_dummy_fe_qpsk_attach(void); -extern struct dvb_frontend* dvb_dummy_fe_qam_attach(void); +struct dvb_frontend *dvb_dummy_fe_ofdm_attach(void); +struct dvb_frontend *dvb_dummy_fe_qpsk_attach(void); +struct dvb_frontend *dvb_dummy_fe_qam_attach(void); #else static inline struct dvb_frontend *dvb_dummy_fe_ofdm_attach(void) { - printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__); + pr_warn("%s: driver disabled by Kconfig\n", __func__); return NULL; } static inline struct dvb_frontend *dvb_dummy_fe_qpsk_attach(void) { - printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__); + pr_warn("%s: driver disabled by Kconfig\n", __func__); return NULL; } static inline struct dvb_frontend *dvb_dummy_fe_qam_attach(void) { - printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__); + pr_warn("%s: driver disabled by Kconfig\n", __func__); return NULL; } #endif /* CONFIG_DVB_DUMMY_FE */ diff --git a/drivers/media/dvb-frontends/lgdt330x.c b/drivers/media/dvb-frontends/lgdt330x.c index 651c8aa75e17..da3a8c5e18d8 100644 --- a/drivers/media/dvb-frontends/lgdt330x.c +++ b/drivers/media/dvb-frontends/lgdt330x.c @@ -922,8 +922,8 @@ struct dvb_frontend *lgdt330x_attach(const struct lgdt330x_config *_config, strscpy(board_info.type, "lgdt330x", sizeof(board_info.type)); board_info.addr = demod_address; board_info.platform_data = &config; - client = i2c_new_device(i2c, &board_info); - if (!client || !client->dev.driver) + client = i2c_new_client_device(i2c, &board_info); + if (!i2c_client_has_driver(client)) return NULL; return lgdt330x_get_dvb_frontend(client); diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c index 3a367a585084..c96f05ff5f2f 100644 --- a/drivers/media/dvb-frontends/m88ds3103.c +++ b/drivers/media/dvb-frontends/m88ds3103.c @@ -1277,8 +1277,8 @@ struct dvb_frontend *m88ds3103_attach(const struct m88ds3103_config *cfg, strscpy(board_info.type, "m88ds3103", I2C_NAME_SIZE); board_info.addr = cfg->i2c_addr; board_info.platform_data = &pdata; - client = i2c_new_device(i2c, &board_info); - if (!client || !client->dev.driver) + client = i2c_new_client_device(i2c, &board_info); + if (!i2c_client_has_driver(client)) return NULL; *tuner_i2c_adapter = pdata.get_i2c_adapter(client); diff --git a/drivers/media/dvb-frontends/ts2020.c b/drivers/media/dvb-frontends/ts2020.c index 6c24d6d0d4c9..234607b02edb 100644 --- a/drivers/media/dvb-frontends/ts2020.c +++ b/drivers/media/dvb-frontends/ts2020.c @@ -519,8 +519,8 @@ struct dvb_frontend *ts2020_attach(struct dvb_frontend *fe, strscpy(board_info.type, "ts2020", I2C_NAME_SIZE); board_info.addr = config->tuner_address; board_info.platform_data = &pdata; - client = i2c_new_device(i2c, &board_info); - if (!client || !client->dev.driver) + client = i2c_new_client_device(i2c, &board_info); + if (!i2c_client_has_driver(client)) return NULL; return fe; diff --git a/drivers/media/i2c/adv748x/adv748x.h b/drivers/media/i2c/adv748x/adv748x.h index 5042f9e94aee..fccb388ce179 100644 --- a/drivers/media/i2c/adv748x/adv748x.h +++ b/drivers/media/i2c/adv748x/adv748x.h @@ -394,10 +394,10 @@ int adv748x_write_block(struct adv748x_state *state, int client_page, #define io_read(s, r) adv748x_read(s, ADV748X_PAGE_IO, r) #define io_write(s, r, v) adv748x_write(s, ADV748X_PAGE_IO, r, v) -#define io_clrset(s, r, m, v) io_write(s, r, (io_read(s, r) & ~m) | v) +#define io_clrset(s, r, m, v) io_write(s, r, (io_read(s, r) & ~(m)) | (v)) #define hdmi_read(s, r) adv748x_read(s, ADV748X_PAGE_HDMI, r) -#define hdmi_read16(s, r, m) (((hdmi_read(s, r) << 8) | hdmi_read(s, r+1)) & m) +#define hdmi_read16(s, r, m) (((hdmi_read(s, r) << 8) | hdmi_read(s, (r)+1)) & (m)) #define hdmi_write(s, r, v) adv748x_write(s, ADV748X_PAGE_HDMI, r, v) #define repeater_read(s, r) adv748x_read(s, ADV748X_PAGE_REPEATER, r) @@ -405,11 +405,11 @@ int adv748x_write_block(struct adv748x_state *state, int client_page, #define sdp_read(s, r) adv748x_read(s, ADV748X_PAGE_SDP, r) #define sdp_write(s, r, v) adv748x_write(s, ADV748X_PAGE_SDP, r, v) -#define sdp_clrset(s, r, m, v) sdp_write(s, r, (sdp_read(s, r) & ~m) | v) +#define sdp_clrset(s, r, m, v) sdp_write(s, r, (sdp_read(s, r) & ~(m)) | (v)) #define cp_read(s, r) adv748x_read(s, ADV748X_PAGE_CP, r) #define cp_write(s, r, v) adv748x_write(s, ADV748X_PAGE_CP, r, v) -#define cp_clrset(s, r, m, v) cp_write(s, r, (cp_read(s, r) & ~m) | v) +#define cp_clrset(s, r, m, v) cp_write(s, r, (cp_read(s, r) & ~(m)) | (v)) #define tx_read(t, r) adv748x_read(t->state, t->page, r) #define tx_write(t, r, v) adv748x_write(t->state, t->page, r, v) diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c index 2dedd6ebb236..09004d928d11 100644 --- a/drivers/media/i2c/adv7604.c +++ b/drivers/media/i2c/adv7604.c @@ -1503,23 +1503,14 @@ static void adv76xx_fill_optional_dv_timings_fields(struct v4l2_subdev *sd, static unsigned int adv7604_read_hdmi_pixelclock(struct v4l2_subdev *sd) { - unsigned int freq; int a, b; a = hdmi_read(sd, 0x06); b = hdmi_read(sd, 0x3b); if (a < 0 || b < 0) return 0; - freq = a * 1000000 + ((b & 0x30) >> 4) * 250000; - if (is_hdmi(sd)) { - /* adjust for deep color mode */ - unsigned bits_per_channel = ((hdmi_read(sd, 0x0b) & 0x60) >> 4) + 8; - - freq = freq * 8 / bits_per_channel; - } - - return freq; + return a * 1000000 + ((b & 0x30) >> 4) * 250000; } static unsigned int adv7611_read_hdmi_pixelclock(struct v4l2_subdev *sd) @@ -1530,9 +1521,28 @@ static unsigned int adv7611_read_hdmi_pixelclock(struct v4l2_subdev *sd) b = hdmi_read(sd, 0x52); if (a < 0 || b < 0) return 0; + return ((a << 1) | (b >> 7)) * 1000000 + (b & 0x7f) * 1000000 / 128; } +static unsigned int adv76xx_read_hdmi_pixelclock(struct v4l2_subdev *sd) +{ + struct adv76xx_state *state = to_state(sd); + const struct adv76xx_chip_info *info = state->info; + unsigned int freq, bits_per_channel, pixelrepetition; + + freq = info->read_hdmi_pixelclock(sd); + if (is_hdmi(sd)) { + /* adjust for deep color mode and pixel repetition */ + bits_per_channel = ((hdmi_read(sd, 0x0b) & 0x60) >> 4) + 8; + pixelrepetition = (hdmi_read(sd, 0x05) & 0x0f) + 1; + + freq = freq * 8 / bits_per_channel / pixelrepetition; + } + + return freq; +} + static int adv76xx_query_dv_timings(struct v4l2_subdev *sd, struct v4l2_dv_timings *timings) { @@ -1579,7 +1589,7 @@ static int adv76xx_query_dv_timings(struct v4l2_subdev *sd, bt->width = w; bt->height = h; - bt->pixelclock = info->read_hdmi_pixelclock(sd); + bt->pixelclock = adv76xx_read_hdmi_pixelclock(sd); bt->hfrontporch = hdmi_read16(sd, 0x20, info->hfrontporch_mask); bt->hsync = hdmi_read16(sd, 0x22, info->hsync_mask); bt->hbackporch = hdmi_read16(sd, 0x24, info->hbackporch_mask); diff --git a/drivers/media/i2c/mt9v032.c b/drivers/media/i2c/mt9v032.c index 4b9b98cf6674..5bd3ae82992f 100644 --- a/drivers/media/i2c/mt9v032.c +++ b/drivers/media/i2c/mt9v032.c @@ -428,10 +428,12 @@ static int mt9v032_enum_mbus_code(struct v4l2_subdev *subdev, struct v4l2_subdev_pad_config *cfg, struct v4l2_subdev_mbus_code_enum *code) { + struct mt9v032 *mt9v032 = to_mt9v032(subdev); + if (code->index > 0) return -EINVAL; - code->code = MEDIA_BUS_FMT_SGRBG10_1X10; + code->code = mt9v032->format.code; return 0; } @@ -439,7 +441,11 @@ static int mt9v032_enum_frame_size(struct v4l2_subdev *subdev, struct v4l2_subdev_pad_config *cfg, struct v4l2_subdev_frame_size_enum *fse) { - if (fse->index >= 3 || fse->code != MEDIA_BUS_FMT_SGRBG10_1X10) + struct mt9v032 *mt9v032 = to_mt9v032(subdev); + + if (fse->index >= 3) + return -EINVAL; + if (mt9v032->format.code != fse->code) return -EINVAL; fse->min_width = MT9V032_WINDOW_WIDTH_DEF / (1 << fse->index); diff --git a/drivers/media/i2c/mt9v111.c b/drivers/media/i2c/mt9v111.c index bb41bea950ac..61ae6a0d5679 100644 --- a/drivers/media/i2c/mt9v111.c +++ b/drivers/media/i2c/mt9v111.c @@ -103,7 +103,7 @@ #define MT9V111_MAX_CLKIN 27000000 /* The default sensor configuration at startup time. */ -static struct v4l2_mbus_framefmt mt9v111_def_fmt = { +static const struct v4l2_mbus_framefmt mt9v111_def_fmt = { .width = 640, .height = 480, .code = MEDIA_BUS_FMT_UYVY8_2X8, diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c index 5e495c833d32..854031f0b64a 100644 --- a/drivers/media/i2c/ov5640.c +++ b/drivers/media/i2c/ov5640.c @@ -189,6 +189,7 @@ struct ov5640_mode_info { u32 vtot; const struct reg_value *reg_data; u32 reg_data_size; + u32 max_fps; }; struct ov5640_ctrls { @@ -544,6 +545,7 @@ static const struct ov5640_mode_info ov5640_mode_init_data = { 0, SUBSAMPLING, 640, 1896, 480, 984, ov5640_init_setting_30fps_VGA, ARRAY_SIZE(ov5640_init_setting_30fps_VGA), + OV5640_30_FPS, }; static const struct ov5640_mode_info @@ -551,39 +553,48 @@ ov5640_mode_data[OV5640_NUM_MODES] = { {OV5640_MODE_QCIF_176_144, SUBSAMPLING, 176, 1896, 144, 984, ov5640_setting_QCIF_176_144, - ARRAY_SIZE(ov5640_setting_QCIF_176_144)}, + ARRAY_SIZE(ov5640_setting_QCIF_176_144), + OV5640_30_FPS}, {OV5640_MODE_QVGA_320_240, SUBSAMPLING, 320, 1896, 240, 984, ov5640_setting_QVGA_320_240, - ARRAY_SIZE(ov5640_setting_QVGA_320_240)}, + ARRAY_SIZE(ov5640_setting_QVGA_320_240), + OV5640_30_FPS}, {OV5640_MODE_VGA_640_480, SUBSAMPLING, 640, 1896, 480, 1080, ov5640_setting_VGA_640_480, - ARRAY_SIZE(ov5640_setting_VGA_640_480)}, + ARRAY_SIZE(ov5640_setting_VGA_640_480), + OV5640_60_FPS}, {OV5640_MODE_NTSC_720_480, SUBSAMPLING, 720, 1896, 480, 984, ov5640_setting_NTSC_720_480, - ARRAY_SIZE(ov5640_setting_NTSC_720_480)}, + ARRAY_SIZE(ov5640_setting_NTSC_720_480), + OV5640_30_FPS}, {OV5640_MODE_PAL_720_576, SUBSAMPLING, 720, 1896, 576, 984, ov5640_setting_PAL_720_576, - ARRAY_SIZE(ov5640_setting_PAL_720_576)}, + ARRAY_SIZE(ov5640_setting_PAL_720_576), + OV5640_30_FPS}, {OV5640_MODE_XGA_1024_768, SUBSAMPLING, 1024, 1896, 768, 1080, ov5640_setting_XGA_1024_768, - ARRAY_SIZE(ov5640_setting_XGA_1024_768)}, + ARRAY_SIZE(ov5640_setting_XGA_1024_768), + OV5640_30_FPS}, {OV5640_MODE_720P_1280_720, SUBSAMPLING, 1280, 1892, 720, 740, ov5640_setting_720P_1280_720, - ARRAY_SIZE(ov5640_setting_720P_1280_720)}, + ARRAY_SIZE(ov5640_setting_720P_1280_720), + OV5640_30_FPS}, {OV5640_MODE_1080P_1920_1080, SCALING, 1920, 2500, 1080, 1120, ov5640_setting_1080P_1920_1080, - ARRAY_SIZE(ov5640_setting_1080P_1920_1080)}, + ARRAY_SIZE(ov5640_setting_1080P_1920_1080), + OV5640_30_FPS}, {OV5640_MODE_QSXGA_2592_1944, SCALING, 2592, 2844, 1944, 1968, ov5640_setting_QSXGA_2592_1944, - ARRAY_SIZE(ov5640_setting_QSXGA_2592_1944)}, + ARRAY_SIZE(ov5640_setting_QSXGA_2592_1944), + OV5640_15_FPS}, }; static int ov5640_init_slave_id(struct ov5640_dev *sensor) @@ -874,7 +885,7 @@ static unsigned long ov5640_calc_sys_clk(struct ov5640_dev *sensor, * We have reached the maximum allowed PLL1 output, * increase sysdiv. */ - if (!rate) + if (!_rate) break; /* @@ -1606,14 +1617,8 @@ ov5640_find_mode(struct ov5640_dev *sensor, enum ov5640_frame_rate fr, (!nearest && (mode->hact != width || mode->vact != height))) return NULL; - /* Only 640x480 can operate at 60fps (for now) */ - if (fr == OV5640_60_FPS && - !(mode->hact == 640 && mode->vact == 480)) - return NULL; - - /* 2592x1944 only works at 15fps max */ - if ((mode->hact == 2592 && mode->vact == 1944) && - fr > OV5640_15_FPS) + /* Check to see if the current mode exceeds the max frame rate */ + if (ov5640_framerates[fr] > ov5640_framerates[mode->max_fps]) return NULL; return mode; diff --git a/drivers/media/i2c/smiapp/smiapp-core.c b/drivers/media/i2c/smiapp/smiapp-core.c index 84f9771b5fed..a80d7701b519 100644 --- a/drivers/media/i2c/smiapp/smiapp-core.c +++ b/drivers/media/i2c/smiapp/smiapp-core.c @@ -413,21 +413,14 @@ static int smiapp_set_ctrl(struct v4l2_ctrl *ctrl) struct smiapp_sensor *sensor = container_of(ctrl->handler, struct smiapp_subdev, ctrl_handler) ->sensor; + struct i2c_client *client = v4l2_get_subdevdata(&sensor->src->sd); + int pm_status; u32 orient = 0; + unsigned int i; int exposure; int rval; switch (ctrl->id) { - case V4L2_CID_ANALOGUE_GAIN: - return smiapp_write( - sensor, - SMIAPP_REG_U16_ANALOGUE_GAIN_CODE_GLOBAL, ctrl->val); - - case V4L2_CID_EXPOSURE: - return smiapp_write( - sensor, - SMIAPP_REG_U16_COARSE_INTEGRATION_TIME, ctrl->val); - case V4L2_CID_HFLIP: case V4L2_CID_VFLIP: if (sensor->streaming) @@ -440,15 +433,10 @@ static int smiapp_set_ctrl(struct v4l2_ctrl *ctrl) orient |= SMIAPP_IMAGE_ORIENTATION_VFLIP; orient ^= sensor->hvflip_inv_mask; - rval = smiapp_write(sensor, SMIAPP_REG_U8_IMAGE_ORIENTATION, - orient); - if (rval < 0) - return rval; smiapp_update_mbus_formats(sensor); - return 0; - + break; case V4L2_CID_VBLANK: exposure = sensor->exposure->val; @@ -461,59 +449,105 @@ static int smiapp_set_ctrl(struct v4l2_ctrl *ctrl) return rval; } - return smiapp_write( - sensor, SMIAPP_REG_U16_FRAME_LENGTH_LINES, - sensor->pixel_array->crop[SMIAPP_PA_PAD_SRC].height - + ctrl->val); - - case V4L2_CID_HBLANK: - return smiapp_write( - sensor, SMIAPP_REG_U16_LINE_LENGTH_PCK, - sensor->pixel_array->crop[SMIAPP_PA_PAD_SRC].width - + ctrl->val); - + break; case V4L2_CID_LINK_FREQ: if (sensor->streaming) return -EBUSY; - return smiapp_pll_update(sensor); - - case V4L2_CID_TEST_PATTERN: { - unsigned int i; + rval = smiapp_pll_update(sensor); + if (rval) + return rval; + return 0; + case V4L2_CID_TEST_PATTERN: for (i = 0; i < ARRAY_SIZE(sensor->test_data); i++) v4l2_ctrl_activate( sensor->test_data[i], ctrl->val == V4L2_SMIAPP_TEST_PATTERN_MODE_SOLID_COLOUR); - return smiapp_write( - sensor, SMIAPP_REG_U16_TEST_PATTERN_MODE, ctrl->val); + break; } + pm_runtime_get_noresume(&client->dev); + pm_status = pm_runtime_get_if_in_use(&client->dev); + pm_runtime_put_noidle(&client->dev); + if (!pm_status) + return 0; + + switch (ctrl->id) { + case V4L2_CID_ANALOGUE_GAIN: + rval = smiapp_write( + sensor, + SMIAPP_REG_U16_ANALOGUE_GAIN_CODE_GLOBAL, ctrl->val); + + break; + case V4L2_CID_EXPOSURE: + rval = smiapp_write( + sensor, + SMIAPP_REG_U16_COARSE_INTEGRATION_TIME, ctrl->val); + + break; + case V4L2_CID_HFLIP: + case V4L2_CID_VFLIP: + rval = smiapp_write(sensor, SMIAPP_REG_U8_IMAGE_ORIENTATION, + orient); + + break; + case V4L2_CID_VBLANK: + rval = smiapp_write( + sensor, SMIAPP_REG_U16_FRAME_LENGTH_LINES, + sensor->pixel_array->crop[SMIAPP_PA_PAD_SRC].height + + ctrl->val); + + break; + case V4L2_CID_HBLANK: + rval = smiapp_write( + sensor, SMIAPP_REG_U16_LINE_LENGTH_PCK, + sensor->pixel_array->crop[SMIAPP_PA_PAD_SRC].width + + ctrl->val); + + break; + case V4L2_CID_TEST_PATTERN: + rval = smiapp_write( + sensor, SMIAPP_REG_U16_TEST_PATTERN_MODE, ctrl->val); + + break; case V4L2_CID_TEST_PATTERN_RED: - return smiapp_write( + rval = smiapp_write( sensor, SMIAPP_REG_U16_TEST_DATA_RED, ctrl->val); + break; case V4L2_CID_TEST_PATTERN_GREENR: - return smiapp_write( + rval = smiapp_write( sensor, SMIAPP_REG_U16_TEST_DATA_GREENR, ctrl->val); + break; case V4L2_CID_TEST_PATTERN_BLUE: - return smiapp_write( + rval = smiapp_write( sensor, SMIAPP_REG_U16_TEST_DATA_BLUE, ctrl->val); + break; case V4L2_CID_TEST_PATTERN_GREENB: - return smiapp_write( + rval = smiapp_write( sensor, SMIAPP_REG_U16_TEST_DATA_GREENB, ctrl->val); + break; case V4L2_CID_PIXEL_RATE: /* For v4l2_ctrl_s_ctrl_int64() used internally. */ - return 0; + rval = 0; + break; default: - return -EINVAL; + rval = -EINVAL; } + + if (pm_status > 0) { + pm_runtime_mark_last_busy(&client->dev); + pm_runtime_put_autosuspend(&client->dev); + } + + return rval; } static const struct v4l2_ctrl_ops smiapp_ctrl_ops = { @@ -1184,10 +1218,6 @@ static int smiapp_power_on(struct device *dev) sleep = SMIAPP_RESET_DELAY(sensor->hwcfg->ext_clk); usleep_range(sleep, sleep); - mutex_lock(&sensor->mutex); - - sensor->active = true; - /* * Failures to respond to the address change command have been noticed. * Those failures seem to be caused by the sensor requiring a longer @@ -1270,24 +1300,9 @@ static int smiapp_power_on(struct device *dev) goto out_cci_addr_fail; } - /* Are we still initialising...? If not, proceed with control setup. */ - if (sensor->pixel_array) { - rval = __v4l2_ctrl_handler_setup( - &sensor->pixel_array->ctrl_handler); - if (rval) - goto out_cci_addr_fail; - - rval = __v4l2_ctrl_handler_setup(&sensor->src->ctrl_handler); - if (rval) - goto out_cci_addr_fail; - } - - mutex_unlock(&sensor->mutex); - return 0; out_cci_addr_fail: - mutex_unlock(&sensor->mutex); gpiod_set_value(sensor->xshutdown, 0); clk_disable_unprepare(sensor->ext_clk); @@ -1305,8 +1320,6 @@ static int smiapp_power_off(struct device *dev) struct smiapp_sensor *sensor = container_of(ssd, struct smiapp_sensor, ssds[0]); - mutex_lock(&sensor->mutex); - /* * Currently power/clock to lens are enable/disabled separately * but they are essentially the same signals. So if the sensor is @@ -1319,10 +1332,6 @@ static int smiapp_power_off(struct device *dev) SMIAPP_REG_U8_SOFTWARE_RESET, SMIAPP_SOFTWARE_RESET); - sensor->active = false; - - mutex_unlock(&sensor->mutex); - gpiod_set_value(sensor->xshutdown, 0); clk_disable_unprepare(sensor->ext_clk); usleep_range(5000, 5000); @@ -1507,6 +1516,30 @@ out: * V4L2 subdev video operations */ +static int smiapp_pm_get_init(struct smiapp_sensor *sensor) +{ + struct i2c_client *client = v4l2_get_subdevdata(&sensor->src->sd); + int rval; + + rval = pm_runtime_get_sync(&client->dev); + if (rval < 0) { + if (rval != -EBUSY && rval != -EAGAIN) + pm_runtime_set_active(&client->dev); + pm_runtime_put_noidle(&client->dev); + + return rval; + } else if (!rval) { + rval = v4l2_ctrl_handler_setup(&sensor->pixel_array-> + ctrl_handler); + if (rval) + return rval; + + return v4l2_ctrl_handler_setup(&sensor->src->ctrl_handler); + } + + return 0; +} + static int smiapp_set_stream(struct v4l2_subdev *subdev, int enable) { struct smiapp_sensor *sensor = to_smiapp_sensor(subdev); @@ -1516,22 +1549,23 @@ static int smiapp_set_stream(struct v4l2_subdev *subdev, int enable) if (sensor->streaming == enable) return 0; - if (enable) { - rval = pm_runtime_get_sync(&client->dev); - if (rval < 0) { - if (rval != -EBUSY && rval != -EAGAIN) - pm_runtime_set_active(&client->dev); - pm_runtime_put(&client->dev); - return rval; - } + if (!enable) { + smiapp_stop_streaming(sensor); + sensor->streaming = false; + pm_runtime_mark_last_busy(&client->dev); + pm_runtime_put_autosuspend(&client->dev); - sensor->streaming = true; + return 0; + } - rval = smiapp_start_streaming(sensor); - if (rval < 0) - sensor->streaming = false; - } else { - rval = smiapp_stop_streaming(sensor); + rval = smiapp_pm_get_init(sensor); + if (rval) + return rval; + + sensor->streaming = true; + + rval = smiapp_start_streaming(sensor); + if (rval < 0) { sensor->streaming = false; pm_runtime_mark_last_busy(&client->dev); pm_runtime_put_autosuspend(&client->dev); @@ -2291,13 +2325,9 @@ smiapp_sysfs_nvm_read(struct device *dev, struct device_attribute *attr, if (!sensor->dev_init_done) return -EBUSY; - rval = pm_runtime_get_sync(&client->dev); - if (rval < 0) { - if (rval != -EBUSY && rval != -EAGAIN) - pm_runtime_set_active(&client->dev); - pm_runtime_put_noidle(&client->dev); + rval = smiapp_pm_get_init(sensor); + if (rval < 0) return -ENODEV; - } rval = smiapp_read_nvm(sensor, buf, PAGE_SIZE); if (rval < 0) { diff --git a/drivers/media/i2c/smiapp/smiapp-regs.c b/drivers/media/i2c/smiapp/smiapp-regs.c index 0470e47c2f7a..ce8c1d47fbf0 100644 --- a/drivers/media/i2c/smiapp/smiapp-regs.c +++ b/drivers/media/i2c/smiapp/smiapp-regs.c @@ -223,9 +223,6 @@ int smiapp_write_no_quirk(struct smiapp_sensor *sensor, u32 reg, u32 val) len != SMIAPP_REG_32BIT) || flags) return -EINVAL; - if (!sensor->active) - return 0; - msg.addr = client->addr; msg.flags = 0; /* Write */ msg.len = 2 + len; diff --git a/drivers/media/i2c/smiapp/smiapp.h b/drivers/media/i2c/smiapp/smiapp.h index 3ab874a5deba..4837d80dc453 100644 --- a/drivers/media/i2c/smiapp/smiapp.h +++ b/drivers/media/i2c/smiapp/smiapp.h @@ -198,7 +198,6 @@ struct smiapp_sensor { u8 hvflip_inv_mask; /* H/VFLIP inversion due to sensor orientation */ u8 frame_skip; - bool active; /* is the sensor powered on? */ u16 embedded_start; /* embedded data start line */ u16 embedded_end; u16 image_start; /* image data start line */ diff --git a/drivers/media/pci/bt8xx/bttv-input.c b/drivers/media/pci/bt8xx/bttv-input.c index 492bc85c2700..41226f1d0e5b 100644 --- a/drivers/media/pci/bt8xx/bttv-input.c +++ b/drivers/media/pci/bt8xx/bttv-input.c @@ -386,7 +386,7 @@ void init_bttv_i2c_ir(struct bttv *btv) if (btv->init_data.name) { info.platform_data = &btv->init_data; - i2c_dev = i2c_new_device(&btv->c.i2c_adap, &info); + i2c_dev = i2c_new_client_device(&btv->c.i2c_adap, &info); } else { /* * The external IR receiver is at i2c address 0x34 (0x35 for @@ -396,9 +396,9 @@ void init_bttv_i2c_ir(struct bttv *btv) * internal. * That's why we probe 0x1a (~0x34) first. CB */ - i2c_dev = i2c_new_probed_device(&btv->c.i2c_adap, &info, addr_list, NULL); + i2c_dev = i2c_new_scanned_device(&btv->c.i2c_adap, &info, addr_list, NULL); } - if (NULL == i2c_dev) + if (IS_ERR(i2c_dev)) return; #if defined(CONFIG_MODULES) && defined(MODULE) diff --git a/drivers/media/pci/cobalt/cobalt-alsa-pcm.c b/drivers/media/pci/cobalt/cobalt-alsa-pcm.c index 38d00935a292..9e7504e3cfd8 100644 --- a/drivers/media/pci/cobalt/cobalt-alsa-pcm.c +++ b/drivers/media/pci/cobalt/cobalt-alsa-pcm.c @@ -9,7 +9,6 @@ #include <linux/init.h> #include <linux/kernel.h> -#include <linux/vmalloc.h> #include <linux/delay.h> #include <media/v4l2-device.h> @@ -238,54 +237,6 @@ static int snd_cobalt_pcm_capture_close(struct snd_pcm_substream *substream) return 0; } -static int snd_cobalt_pcm_ioctl(struct snd_pcm_substream *substream, - unsigned int cmd, void *arg) -{ - return snd_pcm_lib_ioctl(substream, cmd, arg); -} - - -static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs, - size_t size) -{ - struct snd_pcm_runtime *runtime = subs->runtime; - - dprintk("Allocating vbuffer\n"); - if (runtime->dma_area) { - if (runtime->dma_bytes > size) - return 0; - - vfree(runtime->dma_area); - } - runtime->dma_area = vmalloc(size); - if (!runtime->dma_area) - return -ENOMEM; - - runtime->dma_bytes = size; - - return 0; -} - -static int snd_cobalt_pcm_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params) -{ - dprintk("%s called\n", __func__); - - return snd_pcm_alloc_vmalloc_buffer(substream, - params_buffer_bytes(params)); -} - -static int snd_cobalt_pcm_hw_free(struct snd_pcm_substream *substream) -{ - if (substream->runtime->dma_area) { - dprintk("freeing pcm capture region\n"); - vfree(substream->runtime->dma_area); - substream->runtime->dma_area = NULL; - } - - return 0; -} - static int snd_cobalt_pcm_prepare(struct snd_pcm_substream *substream) { struct snd_cobalt_card *cobsc = snd_pcm_substream_chip(substream); @@ -490,36 +441,20 @@ snd_pcm_uframes_t snd_cobalt_pcm_pb_pointer(struct snd_pcm_substream *substream) substream->runtime->buffer_size; } -static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs, - unsigned long offset) -{ - void *pageptr = subs->runtime->dma_area + offset; - - return vmalloc_to_page(pageptr); -} - static const struct snd_pcm_ops snd_cobalt_pcm_capture_ops = { .open = snd_cobalt_pcm_capture_open, .close = snd_cobalt_pcm_capture_close, - .ioctl = snd_cobalt_pcm_ioctl, - .hw_params = snd_cobalt_pcm_hw_params, - .hw_free = snd_cobalt_pcm_hw_free, .prepare = snd_cobalt_pcm_prepare, .trigger = snd_cobalt_pcm_trigger, .pointer = snd_cobalt_pcm_pointer, - .page = snd_pcm_get_vmalloc_page, }; static const struct snd_pcm_ops snd_cobalt_pcm_playback_ops = { .open = snd_cobalt_pcm_playback_open, .close = snd_cobalt_pcm_playback_close, - .ioctl = snd_cobalt_pcm_ioctl, - .hw_params = snd_cobalt_pcm_hw_params, - .hw_free = snd_cobalt_pcm_hw_free, .prepare = snd_cobalt_pcm_pb_prepare, .trigger = snd_cobalt_pcm_pb_trigger, .pointer = snd_cobalt_pcm_pb_pointer, - .page = snd_pcm_get_vmalloc_page, }; int snd_cobalt_pcm_create(struct snd_cobalt_card *cobsc) @@ -555,6 +490,8 @@ int snd_cobalt_pcm_create(struct snd_cobalt_card *cobsc) snd_pcm_set_ops(sp, SNDRV_PCM_STREAM_CAPTURE, &snd_cobalt_pcm_capture_ops); + snd_pcm_set_managed_buffer_all(sp, SNDRV_DMA_TYPE_VMALLOC, + NULL, 0, 0); sp->info_flags = 0; sp->private_data = cobsc; strscpy(sp->name, "cobalt", sizeof(sp->name)); @@ -579,6 +516,8 @@ int snd_cobalt_pcm_create(struct snd_cobalt_card *cobsc) snd_pcm_set_ops(sp, SNDRV_PCM_STREAM_PLAYBACK, &snd_cobalt_pcm_playback_ops); + snd_pcm_set_managed_buffer_all(sp, SNDRV_DMA_TYPE_VMALLOC, + NULL, 0, 0); sp->info_flags = 0; sp->private_data = cobsc; strscpy(sp->name, "cobalt", sizeof(sp->name)); diff --git a/drivers/media/pci/cx18/cx18-alsa-pcm.c b/drivers/media/pci/cx18/cx18-alsa-pcm.c index 13f858c41836..bed28b4b41f7 100644 --- a/drivers/media/pci/cx18/cx18-alsa-pcm.c +++ b/drivers/media/pci/cx18/cx18-alsa-pcm.c @@ -11,7 +11,6 @@ #include <linux/init.h> #include <linux/kernel.h> -#include <linux/vmalloc.h> #include <media/v4l2-device.h> @@ -201,67 +200,6 @@ static int snd_cx18_pcm_capture_close(struct snd_pcm_substream *substream) return 0; } -static int snd_cx18_pcm_ioctl(struct snd_pcm_substream *substream, - unsigned int cmd, void *arg) -{ - struct snd_cx18_card *cxsc = snd_pcm_substream_chip(substream); - int ret; - - snd_cx18_lock(cxsc); - ret = snd_pcm_lib_ioctl(substream, cmd, arg); - snd_cx18_unlock(cxsc); - return ret; -} - - -static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs, - size_t size) -{ - struct snd_pcm_runtime *runtime = subs->runtime; - - dprintk("Allocating vbuffer\n"); - if (runtime->dma_area) { - if (runtime->dma_bytes > size) - return 0; - - vfree(runtime->dma_area); - } - runtime->dma_area = vmalloc(size); - if (!runtime->dma_area) - return -ENOMEM; - - runtime->dma_bytes = size; - - return 0; -} - -static int snd_cx18_pcm_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params) -{ - dprintk("%s called\n", __func__); - - return snd_pcm_alloc_vmalloc_buffer(substream, - params_buffer_bytes(params)); -} - -static int snd_cx18_pcm_hw_free(struct snd_pcm_substream *substream) -{ - struct snd_cx18_card *cxsc = snd_pcm_substream_chip(substream); - unsigned long flags; - unsigned char *dma_area = NULL; - - spin_lock_irqsave(&cxsc->slock, flags); - if (substream->runtime->dma_area) { - dprintk("freeing pcm capture region\n"); - dma_area = substream->runtime->dma_area; - substream->runtime->dma_area = NULL; - } - spin_unlock_irqrestore(&cxsc->slock, flags); - vfree(dma_area); - - return 0; -} - static int snd_cx18_pcm_prepare(struct snd_pcm_substream *substream) { struct snd_cx18_card *cxsc = snd_pcm_substream_chip(substream); @@ -291,24 +229,12 @@ snd_pcm_uframes_t snd_cx18_pcm_pointer(struct snd_pcm_substream *substream) return hwptr_done; } -static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs, - unsigned long offset) -{ - void *pageptr = subs->runtime->dma_area + offset; - - return vmalloc_to_page(pageptr); -} - static const struct snd_pcm_ops snd_cx18_pcm_capture_ops = { .open = snd_cx18_pcm_capture_open, .close = snd_cx18_pcm_capture_close, - .ioctl = snd_cx18_pcm_ioctl, - .hw_params = snd_cx18_pcm_hw_params, - .hw_free = snd_cx18_pcm_hw_free, .prepare = snd_cx18_pcm_prepare, .trigger = snd_cx18_pcm_trigger, .pointer = snd_cx18_pcm_pointer, - .page = snd_pcm_get_vmalloc_page, }; int snd_cx18_pcm_create(struct snd_cx18_card *cxsc) @@ -334,6 +260,7 @@ int snd_cx18_pcm_create(struct snd_cx18_card *cxsc) snd_pcm_set_ops(sp, SNDRV_PCM_STREAM_CAPTURE, &snd_cx18_pcm_capture_ops); + snd_pcm_set_managed_buffer_all(sp, SNDRV_DMA_TYPE_VMALLOC, NULL, 0, 0); sp->info_flags = 0; sp->private_data = cxsc; strscpy(sp->name, cx->card_name, sizeof(sp->name)); diff --git a/drivers/media/pci/cx18/cx18-cards.c b/drivers/media/pci/cx18/cx18-cards.c index cf118760d124..ecbe76f1ca63 100644 --- a/drivers/media/pci/cx18/cx18-cards.c +++ b/drivers/media/pci/cx18/cx18-cards.c @@ -245,7 +245,7 @@ static const struct cx18_card cx18_card_mpc718 = { .type = CX18_CARD_YUAN_MPC718, .name = "Yuan MPC718 MiniPCI DVB-T/Analog", .comment = "Experimenters needed for device to work well.\n" - "\tTo help, mail the ivtv-devel list (www.ivtvdriver.org).\n", + "\tTo help, mail the linux-media list (www.linuxtv.org).\n", .v4l2_capabilities = CX18_CAP_ENCODER, .hw_audio_ctrl = CX18_HW_418_AV, .hw_muxer = CX18_HW_GPIO_MUX, @@ -305,7 +305,7 @@ static const struct cx18_card cx18_card_gotview_dvd3 = { .type = CX18_CARD_GOTVIEW_PCI_DVD3, .name = "GoTView PCI DVD3 Hybrid", .comment = "Experimenters needed for device to work well.\n" - "\tTo help, mail the ivtv-devel list (www.ivtvdriver.org).\n", + "\tTo help, mail the linux-media list (www.linuxtv.org).\n", .v4l2_capabilities = CX18_CAP_ENCODER, .hw_audio_ctrl = CX18_HW_418_AV, .hw_muxer = CX18_HW_GPIO_MUX, @@ -419,7 +419,7 @@ static const struct cx18_card cx18_card_toshiba_qosmio_dvbt = { .type = CX18_CARD_TOSHIBA_QOSMIO_DVBT, .name = "Toshiba Qosmio DVB-T/Analog", .comment = "Experimenters and photos needed for device to work well.\n" - "\tTo help, mail the ivtv-devel list (www.ivtvdriver.org).\n", + "\tTo help, mail the linux-media list (www.linuxtv.org).\n", .v4l2_capabilities = CX18_CAP_ENCODER, .hw_audio_ctrl = CX18_HW_418_AV, .hw_all = CX18_HW_418_AV | CX18_HW_TUNER | CX18_HW_GPIO_RESET_CTRL, @@ -462,7 +462,7 @@ static const struct cx18_card cx18_card_leadtek_pvr2100 = { .type = CX18_CARD_LEADTEK_PVR2100, .name = "Leadtek WinFast PVR2100", .comment = "Experimenters and photos needed for device to work well.\n" - "\tTo help, mail the ivtv-devel list (www.ivtvdriver.org).\n", + "\tTo help, mail the linux-media list (www.linuxtv.org).\n", .v4l2_capabilities = CX18_CAP_ENCODER, .hw_audio_ctrl = CX18_HW_418_AV, .hw_muxer = CX18_HW_GPIO_MUX, diff --git a/drivers/media/pci/cx18/cx18-driver.c b/drivers/media/pci/cx18/cx18-driver.c index 2f1eeeb6e7c7..95aed00f353b 100644 --- a/drivers/media/pci/cx18/cx18-driver.c +++ b/drivers/media/pci/cx18/cx18-driver.c @@ -676,7 +676,7 @@ done: cx->pci_dev->subsystem_device); CX18_ERR("Defaulting to %s card\n", cx->card->name); CX18_ERR("Please mail the vendor/device and subsystem vendor/device IDs and what kind of\n"); - CX18_ERR("card you have to the ivtv-devel mailinglist (www.ivtvdriver.org)\n"); + CX18_ERR("card you have to the linux-media mailinglist (www.linuxtv.org)\n"); CX18_ERR("Prefix your subject line with [UNKNOWN CX18 CARD].\n"); } cx->v4l2_cap = cx->card->v4l2_capabilities; diff --git a/drivers/media/pci/cx18/cx18-i2c.c b/drivers/media/pci/cx18/cx18-i2c.c index 1ef7ccf4a722..a83435245251 100644 --- a/drivers/media/pci/cx18/cx18-i2c.c +++ b/drivers/media/pci/cx18/cx18-i2c.c @@ -88,7 +88,7 @@ static int cx18_i2c_new_ir(struct cx18 *cx, struct i2c_adapter *adap, u32 hw, break; } - return i2c_new_probed_device(adap, &info, addr_list, NULL) == NULL ? + return IS_ERR(i2c_new_scanned_device(adap, &info, addr_list, NULL)) ? -1 : 0; } diff --git a/drivers/media/pci/cx23885/cx23885-alsa.c b/drivers/media/pci/cx23885/cx23885-alsa.c index a8e980c6dacb..df44ed7393a0 100644 --- a/drivers/media/pci/cx23885/cx23885-alsa.c +++ b/drivers/media/pci/cx23885/cx23885-alsa.c @@ -495,7 +495,6 @@ static struct page *snd_cx23885_page(struct snd_pcm_substream *substream, static const struct snd_pcm_ops snd_cx23885_pcm_ops = { .open = snd_cx23885_pcm_open, .close = snd_cx23885_close, - .ioctl = snd_pcm_lib_ioctl, .hw_params = snd_cx23885_hw_params, .hw_free = snd_cx23885_hw_free, .prepare = snd_cx23885_prepare, diff --git a/drivers/media/pci/cx23885/cx23885-cards.c b/drivers/media/pci/cx23885/cx23885-cards.c index 8644205d3cd3..8e5a2c580821 100644 --- a/drivers/media/pci/cx23885/cx23885-cards.c +++ b/drivers/media/pci/cx23885/cx23885-cards.c @@ -801,6 +801,25 @@ struct cx23885_board cx23885_boards[] = { .name = "Hauppauge WinTV-Starburst2", .portb = CX23885_MPEG_DVB, }, + [CX23885_BOARD_AVERMEDIA_CE310B] = { + .name = "AVerMedia CE310B", + .porta = CX23885_ANALOG_VIDEO, + .force_bff = 1, + .input = {{ + .type = CX23885_VMUX_COMPOSITE1, + .vmux = CX25840_VIN1_CH1 | + CX25840_NONE_CH2 | + CX25840_NONE0_CH3, + .amux = CX25840_AUDIO7, + }, { + .type = CX23885_VMUX_SVIDEO, + .vmux = CX25840_VIN8_CH1 | + CX25840_NONE_CH2 | + CX25840_VIN7_CH3 | + CX25840_SVIDEO_ON, + .amux = CX25840_AUDIO7, + } }, + }, }; const unsigned int cx23885_bcount = ARRAY_SIZE(cx23885_boards); @@ -1124,6 +1143,10 @@ struct cx23885_subid cx23885_subids[] = { .subvendor = 0x0070, .subdevice = 0xf02a, .card = CX23885_BOARD_HAUPPAUGE_STARBURST2, + }, { + .subvendor = 0x1461, + .subdevice = 0x3100, + .card = CX23885_BOARD_AVERMEDIA_CE310B, }, }; const unsigned int cx23885_idcount = ARRAY_SIZE(cx23885_subids); @@ -2348,6 +2371,7 @@ void cx23885_card_setup(struct cx23885_dev *dev) case CX23885_BOARD_DVBSKY_T982: case CX23885_BOARD_VIEWCAST_260E: case CX23885_BOARD_VIEWCAST_460E: + case CX23885_BOARD_AVERMEDIA_CE310B: dev->sd_cx25840 = v4l2_i2c_new_subdev(&dev->v4l2_dev, &dev->i2c_bus[2].i2c_adap, "cx25840", 0x88 >> 1, NULL); diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c index 4f386db33a11..494751a067a3 100644 --- a/drivers/media/pci/cx23885/cx23885-dvb.c +++ b/drivers/media/pci/cx23885/cx23885-dvb.c @@ -1159,8 +1159,8 @@ static int dvb_register_ci_mac(struct cx23885_tsport *port) info.addr = 0x40; info.platform_data = &sp2_config; request_module(info.type); - client_ci = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (client_ci == NULL || client_ci->dev.driver == NULL) + client_ci = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_ci)) return -ENODEV; if (!try_module_get(client_ci->dev.driver->owner)) { i2c_unregister_device(client_ci); @@ -1826,8 +1826,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x05; info.platform_data = &tda10071_pdata; request_module("tda10071"); - client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (!client_demod || !client_demod->dev.driver) + client_demod = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -1843,8 +1843,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x0b; info.platform_data = &a8293_pdata; request_module("a8293"); - client_sec = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (!client_sec || !client_sec->dev.driver) + client_sec = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_sec)) goto frontend_detach; if (!try_module_get(client_sec->dev.driver->owner)) { i2c_unregister_device(client_sec); @@ -1864,9 +1864,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x64; info.platform_data = &si2165_pdata; request_module(info.type); - client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (client_demod == NULL || - client_demod->dev.driver == NULL) + client_demod = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -1898,8 +1897,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x05; info.platform_data = &tda10071_pdata; request_module("tda10071"); - client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (!client_demod || !client_demod->dev.driver) + client_demod = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -1915,8 +1914,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x0b; info.platform_data = &a8293_pdata; request_module("a8293"); - client_sec = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (!client_sec || !client_sec->dev.driver) + client_sec = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_sec)) goto frontend_detach; if (!try_module_get(client_sec->dev.driver->owner)) { i2c_unregister_device(client_sec); @@ -1948,9 +1947,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &ts2020_config; request_module(info.type); - client_tuner = i2c_new_device(adapter, &info); - if (client_tuner == NULL || - client_tuner->dev.driver == NULL) + client_tuner = i2c_new_client_device(adapter, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { i2c_unregister_device(client_tuner); @@ -1985,9 +1983,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x64; info.platform_data = &si2168_config; request_module(info.type); - client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (client_demod == NULL || - client_demod->dev.driver == NULL) + client_demod = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -2004,9 +2001,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &si2157_config; request_module(info.type); - client_tuner = i2c_new_device(adapter, &info); - if (client_tuner == NULL || - client_tuner->dev.driver == NULL) + client_tuner = i2c_new_client_device(adapter, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { @@ -2032,8 +2028,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x64; info.platform_data = &si2168_config; request_module(info.type); - client_demod = i2c_new_device(&i2c_bus2->i2c_adap, &info); - if (client_demod == NULL || client_demod->dev.driver == NULL) + client_demod = i2c_new_client_device(&i2c_bus2->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -2050,9 +2046,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &si2157_config; request_module(info.type); - client_tuner = i2c_new_device(adapter, &info); - if (client_tuner == NULL || - client_tuner->dev.driver == NULL) + client_tuner = i2c_new_client_device(adapter, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { i2c_unregister_device(client_tuner); @@ -2080,8 +2075,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &ts2020_config; request_module(info.type); - client_tuner = i2c_new_device(adapter, &info); - if (client_tuner == NULL || client_tuner->dev.driver == NULL) + client_tuner = i2c_new_client_device(adapter, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { i2c_unregister_device(client_tuner); @@ -2129,8 +2124,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x68; info.platform_data = &m88ds3103_pdata; request_module(info.type); - client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (client_demod == NULL || client_demod->dev.driver == NULL) + client_demod = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -2149,8 +2144,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &ts2020_config; request_module(info.type); - client_tuner = i2c_new_device(adapter, &info); - if (client_tuner == NULL || client_tuner->dev.driver == NULL) + client_tuner = i2c_new_client_device(adapter, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { i2c_unregister_device(client_tuner); @@ -2194,8 +2189,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x64; info.platform_data = &si2168_config; request_module(info.type); - client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (client_demod == NULL || client_demod->dev.driver == NULL) + client_demod = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -2212,9 +2207,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &si2157_config; request_module(info.type); - client_tuner = i2c_new_device(adapter, &info); - if (client_tuner == NULL || - client_tuner->dev.driver == NULL) + client_tuner = i2c_new_client_device(adapter, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { i2c_unregister_device(client_tuner); @@ -2245,8 +2239,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x0b; info.platform_data = &a8293_pdata; request_module("a8293"); - client_sec = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (!client_sec || !client_sec->dev.driver) + client_sec = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_sec)) goto frontend_detach; if (!try_module_get(client_sec->dev.driver->owner)) { i2c_unregister_device(client_sec); @@ -2262,8 +2256,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x21; info.platform_data = &m88rs6000t_config; request_module("%s", info.type); - client_tuner = i2c_new_device(adapter, &info); - if (!client_tuner || !client_tuner->dev.driver) + client_tuner = i2c_new_client_device(adapter, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { i2c_unregister_device(client_tuner); @@ -2287,8 +2281,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x64; info.platform_data = &si2168_config; request_module("%s", info.type); - client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info); - if (!client_demod || !client_demod->dev.driver) + client_demod = i2c_new_client_device(&i2c_bus->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -2305,8 +2299,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &si2157_config; request_module("%s", info.type); - client_tuner = i2c_new_device(&i2c_bus2->i2c_adap, &info); - if (!client_tuner || !client_tuner->dev.driver) { + client_tuner = i2c_new_client_device(&i2c_bus2->i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) { module_put(client_demod->dev.driver->owner); i2c_unregister_device(client_demod); port->i2c_client_demod = NULL; @@ -2340,8 +2334,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x64; info.platform_data = &si2168_config; request_module("%s", info.type); - client_demod = i2c_new_device(&dev->i2c_bus[0].i2c_adap, &info); - if (!client_demod || !client_demod->dev.driver) + client_demod = i2c_new_client_device(&dev->i2c_bus[0].i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -2358,8 +2352,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &si2157_config; request_module("%s", info.type); - client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info); - if (!client_tuner || !client_tuner->dev.driver) { + client_tuner = i2c_new_client_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) { module_put(client_demod->dev.driver->owner); i2c_unregister_device(client_demod); port->i2c_client_demod = NULL; @@ -2387,8 +2381,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x66; info.platform_data = &si2168_config; request_module("%s", info.type); - client_demod = i2c_new_device(&dev->i2c_bus[0].i2c_adap, &info); - if (!client_demod || !client_demod->dev.driver) + client_demod = i2c_new_client_device(&dev->i2c_bus[0].i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { i2c_unregister_device(client_demod); @@ -2405,8 +2399,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x62; info.platform_data = &si2157_config; request_module("%s", info.type); - client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info); - if (!client_tuner || !client_tuner->dev.driver) { + client_tuner = i2c_new_client_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) { module_put(client_demod->dev.driver->owner); i2c_unregister_device(client_demod); port->i2c_client_demod = NULL; @@ -2447,8 +2441,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &si2157_config; request_module("%s", info.type); - client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info); - if (!client_tuner || !client_tuner->dev.driver) { + client_tuner = i2c_new_client_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) { module_put(client_demod->dev.driver->owner); i2c_unregister_device(client_demod); port->i2c_client_demod = NULL; @@ -2483,8 +2477,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x62; info.platform_data = &si2157_config; request_module("%s", info.type); - client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info); - if (!client_tuner || !client_tuner->dev.driver) { + client_tuner = i2c_new_client_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) { module_put(client_demod->dev.driver->owner); i2c_unregister_device(client_demod); port->i2c_client_demod = NULL; @@ -2523,8 +2517,8 @@ static int dvb_register(struct cx23885_tsport *port) info.addr = 0x60; info.platform_data = &si2157_config; request_module("%s", info.type); - client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info); - if (!client_tuner || !client_tuner->dev.driver) + client_tuner = i2c_new_client_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) goto frontend_detach; if (!try_module_get(client_tuner->dev.driver->owner)) { diff --git a/drivers/media/pci/cx23885/cx23885-i2c.c b/drivers/media/pci/cx23885/cx23885-i2c.c index 4f327ee9659e..f51fad33dc04 100644 --- a/drivers/media/pci/cx23885/cx23885-i2c.c +++ b/drivers/media/pci/cx23885/cx23885-i2c.c @@ -337,8 +337,8 @@ int cx23885_i2c_register(struct cx23885_i2c *bus) strscpy(info.type, "ir_video", I2C_NAME_SIZE); /* Use quick read command for probe, some IR chips don't * support writes */ - i2c_new_probed_device(&bus->i2c_adap, &info, addr_list, - i2c_probe_func_quick_read); + i2c_new_scanned_device(&bus->i2c_adap, &info, addr_list, + i2c_probe_func_quick_read); } return bus->i2c_rc; diff --git a/drivers/media/pci/cx23885/cx23885-video.c b/drivers/media/pci/cx23885/cx23885-video.c index 8098b15493de..7fc408ee4934 100644 --- a/drivers/media/pci/cx23885/cx23885-video.c +++ b/drivers/media/pci/cx23885/cx23885-video.c @@ -257,7 +257,8 @@ static int cx23885_video_mux(struct cx23885_dev *dev, unsigned int input) (dev->board == CX23885_BOARD_MYGICA_X8507) || (dev->board == CX23885_BOARD_AVERMEDIA_HC81R) || (dev->board == CX23885_BOARD_VIEWCAST_260E) || - (dev->board == CX23885_BOARD_VIEWCAST_460E)) { + (dev->board == CX23885_BOARD_VIEWCAST_460E) || + (dev->board == CX23885_BOARD_AVERMEDIA_CE310B)) { /* Configure audio routing */ v4l2_subdev_call(dev->sd_cx25840, audio, s_routing, INPUT(input)->amux, 0, 0); diff --git a/drivers/media/pci/cx23885/cx23885.h b/drivers/media/pci/cx23885/cx23885.h index a95a2e4c6a0d..c472498e57c4 100644 --- a/drivers/media/pci/cx23885/cx23885.h +++ b/drivers/media/pci/cx23885/cx23885.h @@ -101,6 +101,7 @@ #define CX23885_BOARD_HAUPPAUGE_STARBURST2 59 #define CX23885_BOARD_HAUPPAUGE_QUADHD_DVB_885 60 #define CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC_885 61 +#define CX23885_BOARD_AVERMEDIA_CE310B 62 #define GPIO_0 0x00000001 #define GPIO_1 0x00000002 diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c index c2f2d7c782c7..301616426d8a 100644 --- a/drivers/media/pci/cx25821/cx25821-alsa.c +++ b/drivers/media/pci/cx25821/cx25821-alsa.c @@ -639,7 +639,6 @@ static struct page *snd_cx25821_page(struct snd_pcm_substream *substream, static const struct snd_pcm_ops snd_cx25821_pcm_ops = { .open = snd_cx25821_pcm_open, .close = snd_cx25821_close, - .ioctl = snd_pcm_lib_ioctl, .hw_params = snd_cx25821_hw_params, .hw_free = snd_cx25821_hw_free, .prepare = snd_cx25821_prepare, diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c index e1e71ae293ed..7d7aceecc985 100644 --- a/drivers/media/pci/cx88/cx88-alsa.c +++ b/drivers/media/pci/cx88/cx88-alsa.c @@ -585,7 +585,6 @@ static struct page *snd_cx88_page(struct snd_pcm_substream *substream, static const struct snd_pcm_ops snd_cx88_pcm_ops = { .open = snd_cx88_pcm_open, .close = snd_cx88_close, - .ioctl = snd_pcm_lib_ioctl, .hw_params = snd_cx88_hw_params, .hw_free = snd_cx88_hw_free, .prepare = snd_cx88_prepare, diff --git a/drivers/media/pci/cx88/cx88-input.c b/drivers/media/pci/cx88/cx88-input.c index 589f52d961eb..c7c2acd55266 100644 --- a/drivers/media/pci/cx88/cx88-input.c +++ b/drivers/media/pci/cx88/cx88-input.c @@ -613,7 +613,7 @@ void cx88_i2c_init_ir(struct cx88_core *core) } /* - * We can't call i2c_new_probed_device() because it uses + * We can't call i2c_new_scanned_device() because it uses * quick writes for probing and at least some RC receiver * devices only reply to reads. * Also, Hauppauge XVR needs to be specified, as address 0x71 diff --git a/drivers/media/pci/ivtv/Kconfig b/drivers/media/pci/ivtv/Kconfig index 36c089103cf9..c729e54692c4 100644 --- a/drivers/media/pci/ivtv/Kconfig +++ b/drivers/media/pci/ivtv/Kconfig @@ -24,7 +24,7 @@ config VIDEO_IVTV PCI personal video recorder devices. This is used in devices such as the Hauppauge PVR-150/250/350/500 - cards. There is a driver homepage at <http://www.ivtvdriver.org>. + cards. To compile this driver as a module, choose M here: the module will be called ivtv. @@ -67,8 +67,7 @@ config VIDEO_FB_IVTV This is a framebuffer driver for the Conexant cx23415 MPEG encoder/decoder. - This is used in the Hauppauge PVR-350 card. There is a driver - homepage at <http://www.ivtvdriver.org>. + This is used in the Hauppauge PVR-350 card. To compile this driver as a module, choose M here: the module will be called ivtvfb. diff --git a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c index 9e6019a159f4..8f346d7da9c8 100644 --- a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c +++ b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c @@ -16,8 +16,6 @@ #include "ivtv-alsa.h" #include "ivtv-alsa-pcm.h" -#include <linux/vmalloc.h> - #include <sound/core.h> #include <sound/pcm.h> @@ -206,67 +204,6 @@ static int snd_ivtv_pcm_capture_close(struct snd_pcm_substream *substream) return 0; } -static int snd_ivtv_pcm_ioctl(struct snd_pcm_substream *substream, - unsigned int cmd, void *arg) -{ - struct snd_ivtv_card *itvsc = snd_pcm_substream_chip(substream); - int ret; - - snd_ivtv_lock(itvsc); - ret = snd_pcm_lib_ioctl(substream, cmd, arg); - snd_ivtv_unlock(itvsc); - return ret; -} - - -static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs, - size_t size) -{ - struct snd_pcm_runtime *runtime = subs->runtime; - - dprintk("Allocating vbuffer\n"); - if (runtime->dma_area) { - if (runtime->dma_bytes > size) - return 0; - - vfree(runtime->dma_area); - } - runtime->dma_area = vmalloc(size); - if (!runtime->dma_area) - return -ENOMEM; - - runtime->dma_bytes = size; - - return 0; -} - -static int snd_ivtv_pcm_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params) -{ - dprintk("%s called\n", __func__); - - return snd_pcm_alloc_vmalloc_buffer(substream, - params_buffer_bytes(params)); -} - -static int snd_ivtv_pcm_hw_free(struct snd_pcm_substream *substream) -{ - struct snd_ivtv_card *itvsc = snd_pcm_substream_chip(substream); - unsigned long flags; - unsigned char *dma_area = NULL; - - spin_lock_irqsave(&itvsc->slock, flags); - if (substream->runtime->dma_area) { - dprintk("freeing pcm capture region\n"); - dma_area = substream->runtime->dma_area; - substream->runtime->dma_area = NULL; - } - spin_unlock_irqrestore(&itvsc->slock, flags); - vfree(dma_area); - - return 0; -} - static int snd_ivtv_pcm_prepare(struct snd_pcm_substream *substream) { struct snd_ivtv_card *itvsc = snd_pcm_substream_chip(substream); @@ -296,24 +233,12 @@ snd_pcm_uframes_t snd_ivtv_pcm_pointer(struct snd_pcm_substream *substream) return hwptr_done; } -static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs, - unsigned long offset) -{ - void *pageptr = subs->runtime->dma_area + offset; - - return vmalloc_to_page(pageptr); -} - static const struct snd_pcm_ops snd_ivtv_pcm_capture_ops = { .open = snd_ivtv_pcm_capture_open, .close = snd_ivtv_pcm_capture_close, - .ioctl = snd_ivtv_pcm_ioctl, - .hw_params = snd_ivtv_pcm_hw_params, - .hw_free = snd_ivtv_pcm_hw_free, .prepare = snd_ivtv_pcm_prepare, .trigger = snd_ivtv_pcm_trigger, .pointer = snd_ivtv_pcm_pointer, - .page = snd_pcm_get_vmalloc_page, }; int snd_ivtv_pcm_create(struct snd_ivtv_card *itvsc) @@ -339,6 +264,7 @@ int snd_ivtv_pcm_create(struct snd_ivtv_card *itvsc) snd_pcm_set_ops(sp, SNDRV_PCM_STREAM_CAPTURE, &snd_ivtv_pcm_capture_ops); + snd_pcm_set_managed_buffer_all(sp, SNDRV_DMA_TYPE_VMALLOC, NULL, 0, 0); sp->info_flags = 0; sp->private_data = itvsc; strscpy(sp->name, itv->card_name, sizeof(sp->name)); diff --git a/drivers/media/pci/ivtv/ivtv-driver.c b/drivers/media/pci/ivtv/ivtv-driver.c index 1f79700a6307..b1dde1be6f5e 100644 --- a/drivers/media/pci/ivtv/ivtv-driver.c +++ b/drivers/media/pci/ivtv/ivtv-driver.c @@ -23,7 +23,6 @@ * Driver for the Conexant CX23415/CX23416 chip. * Author: Kevin Thayer (nufan_wfk at yahoo.com) * License: GPL - * http://www.ivtvdriver.org * * ----- * MPG600/MPG160 support by T.Adachi <tadachi@tadachi-net.com> @@ -723,7 +722,7 @@ done: IVTV_ERR(" %s based\n", chipname); IVTV_ERR("Defaulting to %s card\n", itv->card->name); IVTV_ERR("Please mail the vendor/device and subsystem vendor/device IDs and what kind of\n"); - IVTV_ERR("card you have to the ivtv-devel mailinglist (www.ivtvdriver.org)\n"); + IVTV_ERR("card you have to the linux-media mailinglist (www.linuxtv.org)\n"); IVTV_ERR("Prefix your subject line with [UNKNOWN IVTV CARD].\n"); } itv->v4l2_cap = itv->card->v4l2_capabilities; diff --git a/drivers/media/pci/ivtv/ivtv-driver.h b/drivers/media/pci/ivtv/ivtv-driver.h index cafba6b1055d..e5efe525ad7b 100644 --- a/drivers/media/pci/ivtv/ivtv-driver.h +++ b/drivers/media/pci/ivtv/ivtv-driver.h @@ -28,7 +28,6 @@ * Driver for the cx23415/6 chip. * Author: Kevin Thayer (nufan_wfk at yahoo.com) * License: GPL - * http://www.ivtvdriver.org * * ----- * MPG600/MPG160 support by T.Adachi <tadachi@tadachi-net.com> diff --git a/drivers/media/pci/ivtv/ivtv-i2c.c b/drivers/media/pci/ivtv/ivtv-i2c.c index 0772d757a389..982045c4eea8 100644 --- a/drivers/media/pci/ivtv/ivtv-i2c.c +++ b/drivers/media/pci/ivtv/ivtv-i2c.c @@ -208,12 +208,12 @@ static int ivtv_i2c_new_ir(struct ivtv *itv, u32 hw, const char *type, u8 addr) info.platform_data = init_data; strscpy(info.type, type, I2C_NAME_SIZE); - return i2c_new_probed_device(adap, &info, addr_list, NULL) == NULL ? + return IS_ERR(i2c_new_scanned_device(adap, &info, addr_list, NULL)) ? -1 : 0; } /* Instantiate the IR receiver device using probing -- undesirable */ -struct i2c_client *ivtv_i2c_new_ir_legacy(struct ivtv *itv) +void ivtv_i2c_new_ir_legacy(struct ivtv *itv) { struct i2c_board_info info; /* @@ -235,7 +235,7 @@ struct i2c_client *ivtv_i2c_new_ir_legacy(struct ivtv *itv) memset(&info, 0, sizeof(struct i2c_board_info)); strscpy(info.type, "ir_video", I2C_NAME_SIZE); - return i2c_new_probed_device(&itv->i2c_adap, &info, addr_list, NULL); + i2c_new_scanned_device(&itv->i2c_adap, &info, addr_list, NULL); } int ivtv_i2c_register(struct ivtv *itv, unsigned idx) diff --git a/drivers/media/pci/ivtv/ivtv-i2c.h b/drivers/media/pci/ivtv/ivtv-i2c.h index 462f73449a6e..2d9cdaa682c5 100644 --- a/drivers/media/pci/ivtv/ivtv-i2c.h +++ b/drivers/media/pci/ivtv/ivtv-i2c.h @@ -9,7 +9,7 @@ #ifndef IVTV_I2C_H #define IVTV_I2C_H -struct i2c_client *ivtv_i2c_new_ir_legacy(struct ivtv *itv); +void ivtv_i2c_new_ir_legacy(struct ivtv *itv); int ivtv_i2c_register(struct ivtv *itv, unsigned idx); struct v4l2_subdev *ivtv_find_hw(struct ivtv *itv, u32 hw); diff --git a/drivers/media/pci/meye/meye.c b/drivers/media/pci/meye/meye.c index 0e61c81356ef..3a4c29bc0ba5 100644 --- a/drivers/media/pci/meye/meye.c +++ b/drivers/media/pci/meye/meye.c @@ -1266,7 +1266,7 @@ static int vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) buf->flags |= V4L2_BUF_FLAG_DONE; buf->field = V4L2_FIELD_NONE; - buf->timestamp = ns_to_timeval(meye.grab_buffer[index].ts); + v4l2_buffer_set_timestamp(buf, meye.grab_buffer[index].ts); buf->sequence = meye.grab_buffer[index].sequence; buf->memory = V4L2_MEMORY_MMAP; buf->m.offset = index * gbufsize; @@ -1332,7 +1332,7 @@ static int vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) buf->bytesused = meye.grab_buffer[reqnr].size; buf->flags = V4L2_BUF_FLAG_MAPPED | V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; buf->field = V4L2_FIELD_NONE; - buf->timestamp = ns_to_timeval(meye.grab_buffer[reqnr].ts); + v4l2_buffer_set_timestamp(buf, meye.grab_buffer[reqnr].ts); buf->sequence = meye.grab_buffer[reqnr].sequence; buf->memory = V4L2_MEMORY_MMAP; buf->m.offset = reqnr * gbufsize; diff --git a/drivers/media/pci/saa7134/saa7134-alsa.c b/drivers/media/pci/saa7134/saa7134-alsa.c index 0385127dd7ff..544ca57eee75 100644 --- a/drivers/media/pci/saa7134/saa7134-alsa.c +++ b/drivers/media/pci/saa7134/saa7134-alsa.c @@ -865,7 +865,6 @@ static struct page *snd_card_saa7134_page(struct snd_pcm_substream *substream, static const struct snd_pcm_ops snd_card_saa7134_capture_ops = { .open = snd_card_saa7134_capture_open, .close = snd_card_saa7134_capture_close, - .ioctl = snd_pcm_lib_ioctl, .hw_params = snd_card_saa7134_hw_params, .hw_free = snd_card_saa7134_hw_free, .prepare = snd_card_saa7134_capture_prepare, diff --git a/drivers/media/pci/saa7164/saa7164-dvb.c b/drivers/media/pci/saa7164/saa7164-dvb.c index 05ab4734ea67..bf8c2bb8852e 100644 --- a/drivers/media/pci/saa7164/saa7164-dvb.c +++ b/drivers/media/pci/saa7164/saa7164-dvb.c @@ -116,8 +116,8 @@ static int si2157_attach(struct saa7164_port *port, struct i2c_adapter *adapter, request_module(bi.type); - tuner = i2c_new_device(adapter, &bi); - if (tuner == NULL || tuner->dev.driver == NULL) + tuner = i2c_new_client_device(adapter, &bi); + if (!i2c_client_has_driver(tuner)) return -ENODEV; if (!try_module_get(tuner->dev.driver->owner)) { @@ -637,9 +637,8 @@ int saa7164_dvb_register(struct saa7164_port *port) info.addr = 0xc8 >> 1; info.platform_data = &si2168_config; request_module(info.type); - client_demod = i2c_new_device(&dev->i2c_bus[2].i2c_adap, - &info); - if (!client_demod || !client_demod->dev.driver) + client_demod = i2c_new_client_device(&dev->i2c_bus[2].i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { @@ -657,9 +656,8 @@ int saa7164_dvb_register(struct saa7164_port *port) info.addr = 0xc0 >> 1; info.platform_data = &si2157_config; request_module(info.type); - client_tuner = i2c_new_device(&dev->i2c_bus[0].i2c_adap, - &info); - if (!client_tuner || !client_tuner->dev.driver) { + client_tuner = i2c_new_client_device(&dev->i2c_bus[0].i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) { module_put(client_demod->dev.driver->owner); i2c_unregister_device(client_demod); goto frontend_detach; @@ -682,9 +680,8 @@ int saa7164_dvb_register(struct saa7164_port *port) info.addr = 0xcc >> 1; info.platform_data = &si2168_config; request_module(info.type); - client_demod = i2c_new_device(&dev->i2c_bus[2].i2c_adap, - &info); - if (!client_demod || !client_demod->dev.driver) + client_demod = i2c_new_client_device(&dev->i2c_bus[2].i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto frontend_detach; if (!try_module_get(client_demod->dev.driver->owner)) { @@ -702,9 +699,8 @@ int saa7164_dvb_register(struct saa7164_port *port) info.addr = 0xc0 >> 1; info.platform_data = &si2157_config; request_module(info.type); - client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, - &info); - if (!client_tuner || !client_tuner->dev.driver) { + client_tuner = i2c_new_client_device(&dev->i2c_bus[1].i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) { module_put(client_demod->dev.driver->owner); i2c_unregister_device(client_demod); goto frontend_detach; diff --git a/drivers/media/pci/smipcie/smipcie-main.c b/drivers/media/pci/smipcie/smipcie-main.c index 1fb78462e081..9ca0fc3e6f80 100644 --- a/drivers/media/pci/smipcie/smipcie-main.c +++ b/drivers/media/pci/smipcie/smipcie-main.c @@ -484,8 +484,8 @@ static struct i2c_client *smi_add_i2c_client(struct i2c_adapter *adapter, struct i2c_client *client; request_module(info->type); - client = i2c_new_device(adapter, info); - if (client == NULL || client->dev.driver == NULL) + client = i2c_new_client_device(adapter, info); + if (!i2c_client_has_driver(client)) goto err_add_i2c_client; if (!try_module_get(client->dev.driver->owner)) { diff --git a/drivers/media/pci/solo6x10/solo6x10-g723.c b/drivers/media/pci/solo6x10/solo6x10-g723.c index eaa57d835ea8..d6d16e8fd997 100644 --- a/drivers/media/pci/solo6x10/solo6x10-g723.c +++ b/drivers/media/pci/solo6x10/solo6x10-g723.c @@ -97,17 +97,6 @@ void solo_g723_isr(struct solo_dev *solo_dev) } } -static int snd_solo_hw_params(struct snd_pcm_substream *ss, - struct snd_pcm_hw_params *hw_params) -{ - return snd_pcm_lib_malloc_pages(ss, params_buffer_bytes(hw_params)); -} - -static int snd_solo_hw_free(struct snd_pcm_substream *ss) -{ - return snd_pcm_lib_free_pages(ss); -} - static const struct snd_pcm_hardware snd_solo_pcm_hw = { .info = (SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_INTERLEAVED | @@ -270,9 +259,6 @@ static int snd_solo_pcm_copy_kernel(struct snd_pcm_substream *ss, int channel, static const struct snd_pcm_ops snd_solo_pcm_ops = { .open = snd_solo_pcm_open, .close = snd_solo_pcm_close, - .ioctl = snd_pcm_lib_ioctl, - .hw_params = snd_solo_hw_params, - .hw_free = snd_solo_hw_free, .prepare = snd_solo_pcm_prepare, .trigger = snd_solo_pcm_trigger, .pointer = snd_solo_pcm_pointer, @@ -351,11 +337,11 @@ static int solo_snd_pcm_init(struct solo_dev *solo_dev) ss; ss = ss->next, i++) sprintf(ss->name, "Camera #%d Audio", i); - snd_pcm_lib_preallocate_pages_for_all(pcm, - SNDRV_DMA_TYPE_CONTINUOUS, - NULL, - G723_PERIOD_BYTES * PERIODS, - G723_PERIOD_BYTES * PERIODS); + snd_pcm_set_managed_buffer_all(pcm, + SNDRV_DMA_TYPE_CONTINUOUS, + NULL, + G723_PERIOD_BYTES * PERIODS, + G723_PERIOD_BYTES * PERIODS); solo_dev->snd_pcm = pcm; diff --git a/drivers/media/pci/tw686x/tw686x-audio.c b/drivers/media/pci/tw686x/tw686x-audio.c index 7786e51d19ae..54144e23a487 100644 --- a/drivers/media/pci/tw686x/tw686x-audio.c +++ b/drivers/media/pci/tw686x/tw686x-audio.c @@ -78,17 +78,6 @@ void tw686x_audio_irq(struct tw686x_dev *dev, unsigned long requests, } } -static int tw686x_pcm_hw_params(struct snd_pcm_substream *ss, - struct snd_pcm_hw_params *hw_params) -{ - return snd_pcm_lib_malloc_pages(ss, params_buffer_bytes(hw_params)); -} - -static int tw686x_pcm_hw_free(struct snd_pcm_substream *ss) -{ - return snd_pcm_lib_free_pages(ss); -} - /* * Audio parameters are global and shared among all * capture channels. The driver prevents changes to @@ -269,9 +258,6 @@ static snd_pcm_uframes_t tw686x_pcm_pointer(struct snd_pcm_substream *ss) static const struct snd_pcm_ops tw686x_pcm_ops = { .open = tw686x_pcm_open, .close = tw686x_pcm_close, - .ioctl = snd_pcm_lib_ioctl, - .hw_params = tw686x_pcm_hw_params, - .hw_free = tw686x_pcm_hw_free, .prepare = tw686x_pcm_prepare, .trigger = tw686x_pcm_trigger, .pointer = tw686x_pcm_pointer, @@ -298,7 +284,7 @@ static int tw686x_snd_pcm_init(struct tw686x_dev *dev) ss; ss = ss->next, i++) snprintf(ss->name, sizeof(ss->name), "vch%u audio", i); - snd_pcm_lib_preallocate_pages_for_all(pcm, + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV, &dev->pci_dev->dev, TW686X_AUDIO_PAGE_MAX * AUDIO_DMA_SIZE_MAX, diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index e84f35d3a68e..f65e98d3adf2 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -151,7 +151,7 @@ source "drivers/media/platform/sunxi/Kconfig" config VIDEO_TI_CAL tristate "TI CAL (Camera Adaptation Layer) driver" depends on VIDEO_DEV && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API - depends on SOC_DRA7XX || COMPILE_TEST + depends on SOC_DRA7XX || ARCH_K3 || COMPILE_TEST select VIDEOBUF2_DMA_CONTIG select V4L2_FWNODE help @@ -200,7 +200,7 @@ config VIDEO_IMX_PXP config VIDEO_MEDIATEK_JPEG tristate "Mediatek JPEG Codec driver" - depends on MTK_IOMMU_V1 || COMPILE_TEST + depends on MTK_IOMMU_V1 || MTK_IOMMU || COMPILE_TEST depends on VIDEO_DEV && VIDEO_V4L2 depends on ARCH_MEDIATEK || COMPILE_TEST select VIDEOBUF2_DMA_CONTIG diff --git a/drivers/media/platform/atmel/atmel-isc-base.c b/drivers/media/platform/atmel/atmel-isc-base.c index c1c776b348a9..d7669a03e98e 100644 --- a/drivers/media/platform/atmel/atmel-isc-base.c +++ b/drivers/media/platform/atmel/atmel-isc-base.c @@ -73,6 +73,9 @@ const struct isc_format controller_formats[] = { { .fourcc = V4L2_PIX_FMT_GREY, }, + { + .fourcc = V4L2_PIX_FMT_Y10, + }, }; /* This is a list of formats that the ISC can receive as *input* */ @@ -164,6 +167,12 @@ struct isc_format formats_list[] = { .mbus_code = MEDIA_BUS_FMT_RGB565_2X8_LE, .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, }, + { + .fourcc = V4L2_PIX_FMT_Y10, + .mbus_code = MEDIA_BUS_FMT_Y10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + }, + }; /* Gamma table with gamma 1/2.2 */ @@ -211,6 +220,10 @@ const u32 isc_gamma_table[GAMMA_MAX + 1][GAMMA_ENTRIES] = { #define ISC_IS_FORMAT_RAW(mbus_code) \ (((mbus_code) & 0xf000) == 0x3000) +#define ISC_IS_FORMAT_GREY(mbus_code) \ + (((mbus_code) == MEDIA_BUS_FMT_Y10_1X10) | \ + (((mbus_code) == MEDIA_BUS_FMT_Y8_1X8))) + static inline void isc_update_awb_ctrls(struct isc_device *isc) { struct isc_ctrls *ctrls = &isc->ctrls; @@ -1003,6 +1016,7 @@ static int isc_try_validate_formats(struct isc_device *isc) rgb = true; break; case V4L2_PIX_FMT_GREY: + case V4L2_PIX_FMT_Y10: ret = 0; grey = true; break; @@ -1010,34 +1024,29 @@ static int isc_try_validate_formats(struct isc_device *isc) /* any other different formats are not supported */ ret = -EINVAL; } - - /* we cannot output RAW/Grey if we do not receive RAW */ - if ((bayer || grey) && - !ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) - return -EINVAL; - v4l2_dbg(1, debug, &isc->v4l2_dev, "Format validation, requested rgb=%u, yuv=%u, grey=%u, bayer=%u\n", rgb, yuv, grey, bayer); + /* we cannot output RAW if we do not receive RAW */ + if ((bayer) && !ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) + return -EINVAL; + + /* we cannot output GREY if we do not receive RAW/GREY */ + if (grey && !ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code) && + !ISC_IS_FORMAT_GREY(isc->try_config.sd_format->mbus_code)) + return -EINVAL; + return ret; } /* * Configures the RLP and DMA modules, depending on the output format * configured for the ISC. - * If direct_dump == true, just dump raw data 8 bits. + * If direct_dump == true, just dump raw data 8/16 bits depending on format. */ static int isc_try_configure_rlp_dma(struct isc_device *isc, bool direct_dump) { - if (direct_dump) { - isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT8; - isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8; - isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; - isc->try_config.bpp = 16; - return 0; - } - switch (isc->try_config.fourcc) { case V4L2_PIX_FMT_SBGGR8: case V4L2_PIX_FMT_SGBRG8: @@ -1115,9 +1124,23 @@ static int isc_try_configure_rlp_dma(struct isc_device *isc, bool direct_dump) isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; isc->try_config.bpp = 8; break; + case V4L2_PIX_FMT_Y10: + isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DATY10; + isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16; + isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; + isc->try_config.bpp = 16; + break; default: return -EINVAL; } + + if (direct_dump) { + isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT8; + isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8; + isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; + return 0; + } + return 0; } @@ -1187,13 +1210,44 @@ static int isc_try_configure_pipeline(struct isc_device *isc) return 0; } +static void isc_try_fse(struct isc_device *isc, + struct v4l2_subdev_pad_config *pad_cfg) +{ + int ret; + struct v4l2_subdev_frame_size_enum fse = {}; + + /* + * If we do not know yet which format the subdev is using, we cannot + * do anything. + */ + if (!isc->try_config.sd_format) + return; + + fse.code = isc->try_config.sd_format->mbus_code; + fse.which = V4L2_SUBDEV_FORMAT_TRY; + + ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size, + pad_cfg, &fse); + /* + * Attempt to obtain format size from subdev. If not available, + * just use the maximum ISC can receive. + */ + if (ret) { + pad_cfg->try_crop.width = ISC_MAX_SUPPORT_WIDTH; + pad_cfg->try_crop.height = ISC_MAX_SUPPORT_HEIGHT; + } else { + pad_cfg->try_crop.width = fse.max_width; + pad_cfg->try_crop.height = fse.max_height; + } +} + static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f, u32 *code) { int i; struct isc_format *sd_fmt = NULL, *direct_fmt = NULL; struct v4l2_pix_format *pixfmt = &f->fmt.pix; - struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_pad_config pad_cfg = {}; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -1290,6 +1344,9 @@ static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f, if (ret) goto isc_try_fmt_err; + /* Obtain frame sizes if possible to have crop requirements ready */ + isc_try_fse(isc, &pad_cfg); + v4l2_fill_mbus_format(&format.format, pixfmt, mbus_code); ret = v4l2_subdev_call(isc->current_subdev->sd, pad, set_fmt, &pad_cfg, &format); @@ -1414,6 +1471,7 @@ static int isc_enum_framesizes(struct file *file, void *fh, { struct isc_device *isc = video_drvdata(file); struct v4l2_subdev_frame_size_enum fse = { + .code = isc->config.sd_format->mbus_code, .index = fsize->index, .which = V4L2_SUBDEV_FORMAT_ACTIVE, }; @@ -1436,8 +1494,6 @@ static int isc_enum_framesizes(struct file *file, void *fh, if (ret) return ret; - fse.code = isc->config.sd_format->mbus_code; - fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE; fsize->discrete.width = fse.max_width; fsize->discrete.height = fse.max_height; @@ -1450,6 +1506,7 @@ static int isc_enum_frameintervals(struct file *file, void *fh, { struct isc_device *isc = video_drvdata(file); struct v4l2_subdev_frame_interval_enum fie = { + .code = isc->config.sd_format->mbus_code, .index = fival->index, .width = fival->width, .height = fival->height, @@ -1474,7 +1531,6 @@ static int isc_enum_frameintervals(struct file *file, void *fh, if (ret) return ret; - fie.code = isc->config.sd_format->mbus_code; fival->type = V4L2_FRMIVAL_TYPE_DISCRETE; fival->discrete = fie.interval; diff --git a/drivers/media/platform/atmel/atmel-isi.c b/drivers/media/platform/atmel/atmel-isi.c index 428f117caa59..963dfd6e750e 100644 --- a/drivers/media/platform/atmel/atmel-isi.c +++ b/drivers/media/platform/atmel/atmel-isi.c @@ -148,7 +148,8 @@ static void configure_geometry(struct atmel_isi *isi) u32 fourcc = isi->current_fmt->fourcc; isi->enable_preview_path = fourcc == V4L2_PIX_FMT_RGB565 || - fourcc == V4L2_PIX_FMT_RGB32; + fourcc == V4L2_PIX_FMT_RGB32 || + fourcc == V4L2_PIX_FMT_Y16; /* According to sensor's output format to set cfg2 */ cfg2 = isi->current_fmt->swap; @@ -554,12 +555,36 @@ static const struct isi_format *find_format_by_fourcc(struct atmel_isi *isi, return NULL; } +static void isi_try_fse(struct atmel_isi *isi, const struct isi_format *isi_fmt, + struct v4l2_subdev_pad_config *pad_cfg) +{ + int ret; + struct v4l2_subdev_frame_size_enum fse = { + .code = isi_fmt->mbus_code, + .which = V4L2_SUBDEV_FORMAT_TRY, + }; + + ret = v4l2_subdev_call(isi->entity.subdev, pad, enum_frame_size, + pad_cfg, &fse); + /* + * Attempt to obtain format size from subdev. If not available, + * just use the maximum ISI can receive. + */ + if (ret) { + pad_cfg->try_crop.width = MAX_SUPPORT_WIDTH; + pad_cfg->try_crop.height = MAX_SUPPORT_HEIGHT; + } else { + pad_cfg->try_crop.width = fse.max_width; + pad_cfg->try_crop.height = fse.max_height; + } +} + static int isi_try_fmt(struct atmel_isi *isi, struct v4l2_format *f, const struct isi_format **current_fmt) { const struct isi_format *isi_fmt; struct v4l2_pix_format *pixfmt = &f->fmt.pix; - struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_pad_config pad_cfg = {}; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -576,6 +601,9 @@ static int isi_try_fmt(struct atmel_isi *isi, struct v4l2_format *f, pixfmt->height = clamp(pixfmt->height, 0U, MAX_SUPPORT_HEIGHT); v4l2_fill_mbus_format(&format.format, pixfmt, isi_fmt->mbus_code); + + isi_try_fse(isi, isi_fmt, &pad_cfg); + ret = v4l2_subdev_call(isi->entity.subdev, pad, set_fmt, &pad_cfg, &format); if (ret < 0) @@ -990,6 +1018,16 @@ static const struct isi_format isi_formats[] = { .mbus_code = MEDIA_BUS_FMT_VYUY8_2X8, .bpp = 2, .swap = ISI_CFG2_YCC_SWAP_MODE_1, + }, { + .fourcc = V4L2_PIX_FMT_GREY, + .mbus_code = MEDIA_BUS_FMT_Y10_1X10, + .bpp = 1, + .swap = ISI_CFG2_GS_MODE_2_PIXEL | ISI_CFG2_GRAYSCALE, + }, { + .fourcc = V4L2_PIX_FMT_Y16, + .mbus_code = MEDIA_BUS_FMT_Y10_1X10, + .bpp = 2, + .swap = ISI_CFG2_GS_MODE_2_PIXEL | ISI_CFG2_GRAYSCALE, }, }; diff --git a/drivers/media/platform/atmel/atmel-isi.h b/drivers/media/platform/atmel/atmel-isi.h index 47a9108dba55..7ad3895a2c87 100644 --- a/drivers/media/platform/atmel/atmel-isi.h +++ b/drivers/media/platform/atmel/atmel-isi.h @@ -62,6 +62,8 @@ #define ISI_CFG1_THMASK_BEATS_16 (2 << 13) /* Bitfields in CFG2 */ +#define ISI_CFG2_GS_MODE_2_PIXEL (0 << 11) +#define ISI_CFG2_GS_MODE_1_PIXEL (1 << 11) #define ISI_CFG2_GRAYSCALE (1 << 13) #define ISI_CFG2_COL_SPACE_YCbCr (0 << 15) #define ISI_CFG2_COL_SPACE_RGB (1 << 15) diff --git a/drivers/media/platform/coda/coda-bit.c b/drivers/media/platform/coda/coda-bit.c index 00c7bed3dd57..3443396ba5f3 100644 --- a/drivers/media/platform/coda/coda-bit.c +++ b/drivers/media/platform/coda/coda-bit.c @@ -1629,6 +1629,9 @@ static void coda_finish_encode(struct coda_ctx *ctx) struct coda_dev *dev = ctx->dev; u32 wr_ptr, start_ptr; + if (ctx->aborting) + return; + /* * Lock to make sure that an encoder stop command running in parallel * will either already have marked src_buf as last, or it will wake up @@ -2165,16 +2168,21 @@ static int coda_prepare_decode(struct coda_ctx *ctx) } else { if (dev->devtype->product == CODA_960) { /* - * The CODA960 seems to have an internal list of - * buffers with 64 entries that includes the - * registered frame buffers as well as the rotator - * buffer output. - * - * ROT_INDEX needs to be < 0x40, but > - * ctx->num_internal_frames. + * It was previously assumed that the CODA960 has an + * internal list of 64 buffer entries that contains + * both the registered internal frame buffers as well + * as the rotator buffer output, and that the ROT_INDEX + * register must be set to a value between the last + * internal frame buffers' index and 64. + * At least on firmware version 3.1.1 it turns out that + * setting ROT_INDEX to any value >= 32 causes CODA + * hangups that it can not recover from with the SRC VPU + * reset. + * It does appear to work however, to just set it to a + * fixed value in the [ctx->num_internal_frames, 31] + * range, for example CODA_MAX_FRAMEBUFFERS. */ - coda_write(dev, - CODA_MAX_FRAMEBUFFERS + dst_buf->vb2_buf.index, + coda_write(dev, CODA_MAX_FRAMEBUFFERS, CODA9_CMD_DEC_PIC_ROT_INDEX); reg_addr = CODA9_CMD_DEC_PIC_ROT_ADDR_Y; @@ -2266,6 +2274,9 @@ static void coda_finish_decode(struct coda_ctx *ctx) int err_vdoa = 0; u32 val; + if (ctx->aborting) + return; + /* Update kfifo out pointer from coda bitstream read pointer */ coda_kfifo_sync_from_device(ctx); diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c index 94fb4d2ecc43..acff10ad257a 100644 --- a/drivers/media/platform/coda/coda-common.c +++ b/drivers/media/platform/coda/coda-common.c @@ -155,6 +155,7 @@ static const struct coda_codec coda7_codecs[] = { static const struct coda_codec coda9_codecs[] = { CODA_CODEC(CODA9_MODE_ENCODE_H264, V4L2_PIX_FMT_YUV420, V4L2_PIX_FMT_H264, 1920, 1088), CODA_CODEC(CODA9_MODE_ENCODE_MP4, V4L2_PIX_FMT_YUV420, V4L2_PIX_FMT_MPEG4, 1920, 1088), + CODA_CODEC(CODA9_MODE_ENCODE_MJPG, V4L2_PIX_FMT_YUV420, V4L2_PIX_FMT_JPEG, 8192, 8192), CODA_CODEC(CODA9_MODE_DECODE_H264, V4L2_PIX_FMT_H264, V4L2_PIX_FMT_YUV420, 1920, 1088), CODA_CODEC(CODA9_MODE_DECODE_MP2, V4L2_PIX_FMT_MPEG2, V4L2_PIX_FMT_YUV420, 1920, 1088), CODA_CODEC(CODA9_MODE_DECODE_MP4, V4L2_PIX_FMT_MPEG4, V4L2_PIX_FMT_YUV420, 1920, 1088), @@ -235,6 +236,22 @@ static const struct coda_video_device coda_bit_jpeg_decoder = { }, }; +static const struct coda_video_device coda9_jpeg_encoder = { + .name = "coda-jpeg-encoder", + .type = CODA_INST_ENCODER, + .ops = &coda9_jpeg_encode_ops, + .direct = true, + .src_formats = { + V4L2_PIX_FMT_NV12, + V4L2_PIX_FMT_YUV420, + V4L2_PIX_FMT_YVU420, + V4L2_PIX_FMT_YUV422P, + }, + .dst_formats = { + V4L2_PIX_FMT_JPEG, + }, +}; + static const struct coda_video_device *codadx6_video_devices[] = { &coda_bit_encoder, }; @@ -252,6 +269,7 @@ static const struct coda_video_device *coda7_video_devices[] = { }; static const struct coda_video_device *coda9_video_devices[] = { + &coda9_jpeg_encoder, &coda_bit_encoder, &coda_bit_decoder, }; @@ -721,7 +739,8 @@ static int coda_s_fmt(struct coda_ctx *ctx, struct v4l2_format *f, ctx->tiled_map_type = GDI_TILED_FRAME_MB_RASTER_MAP; break; case V4L2_PIX_FMT_NV12: - if (!disable_tiling && ctx->dev->devtype->product == CODA_960) { + if (!disable_tiling && ctx->use_bit && + ctx->dev->devtype->product == CODA_960) { ctx->tiled_map_type = GDI_TILED_FRAME_MB_RASTER_MAP; break; } @@ -1421,7 +1440,7 @@ static void coda_pic_run_work(struct work_struct *work) if (ctx->ops->run_timeout) ctx->ops->run_timeout(ctx); - } else if (!ctx->aborting) { + } else { ctx->ops->finish_run(ctx); } @@ -1787,7 +1806,7 @@ static void coda_buf_queue(struct vb2_buffer *vb) coda_queue_source_change_event(ctx); } } else { - if (ctx->inst_type == CODA_INST_ENCODER && + if ((ctx->inst_type == CODA_INST_ENCODER || !ctx->use_bit) && vq->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) vbuf->sequence = ctx->qsequence++; v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf); @@ -2984,10 +3003,8 @@ static int coda_probe(struct platform_device *pdev) irq = platform_get_irq_byname(pdev, "bit"); if (irq < 0) irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "failed to get irq resource\n"); + if (irq < 0) return irq; - } ret = devm_request_irq(&pdev->dev, irq, coda_irq_handler, 0, dev_name(&pdev->dev), dev); @@ -2996,6 +3013,22 @@ static int coda_probe(struct platform_device *pdev) return ret; } + /* JPEG IRQ */ + if (dev->devtype->product == CODA_960) { + irq = platform_get_irq_byname(pdev, "jpeg"); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + coda9_jpeg_irq_handler, + IRQF_ONESHOT, CODA_NAME " jpeg", + dev); + if (ret < 0) { + dev_err(&pdev->dev, "failed to request jpeg irq\n"); + return ret; + } + } + dev->rstc = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); if (IS_ERR(dev->rstc)) { diff --git a/drivers/media/platform/coda/coda-jpeg.c b/drivers/media/platform/coda/coda-jpeg.c index bf61a3ecc580..92234fd1f4fd 100644 --- a/drivers/media/platform/coda/coda-jpeg.c +++ b/drivers/media/platform/coda/coda-jpeg.c @@ -5,46 +5,68 @@ * Copyright (C) 2014 Philipp Zabel, Pengutronix */ +#include <asm/unaligned.h> +#include <linux/irqreturn.h> #include <linux/kernel.h> +#include <linux/ktime.h> +#include <linux/slab.h> #include <linux/swab.h> +#include <linux/videodev2.h> + +#include <media/v4l2-common.h> +#include <media/v4l2-fh.h> +#include <media/v4l2-mem2mem.h> +#include <media/videobuf2-core.h> +#include <media/videobuf2-dma-contig.h> #include "coda.h" #include "trace.h" #define SOI_MARKER 0xffd8 +#define DRI_MARKER 0xffdd +#define DQT_MARKER 0xffdb +#define DHT_MARKER 0xffc4 +#define SOF_MARKER 0xffc0 #define EOI_MARKER 0xffd9 +enum { + CODA9_JPEG_FORMAT_420, + CODA9_JPEG_FORMAT_422, + CODA9_JPEG_FORMAT_224, + CODA9_JPEG_FORMAT_444, + CODA9_JPEG_FORMAT_400, +}; + +#define CODA9_JPEG_ENC_HUFF_DATA_SIZE (256 + 256 + 16 + 16) + /* * Typical Huffman tables for 8-bit precision luminance and * chrominance from JPEG ITU-T.81 (ISO/IEC 10918-1) Annex K.3 */ -static const unsigned char luma_dc_bits[16] = { +static const unsigned char luma_dc[16 + 12] = { + /* bits */ 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const unsigned char luma_dc_value[12] = { + /* values */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, }; -static const unsigned char chroma_dc_bits[16] = { +static const unsigned char chroma_dc[16 + 12] = { + /* bits */ 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const unsigned char chroma_dc_value[12] = { + /* values */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, }; -static const unsigned char luma_ac_bits[16] = { +static const unsigned char luma_ac[16 + 162 + 2] = { + /* bits */ 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, -}; - -static const unsigned char luma_ac_value[162 + 2] = { + /* values */ 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, @@ -68,12 +90,11 @@ static const unsigned char luma_ac_value[162 + 2] = { 0xf9, 0xfa, /* padded to 32-bit */ }; -static const unsigned char chroma_ac_bits[16] = { +static const unsigned char chroma_ac[16 + 162 + 2] = { + /* bits */ 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02, 0x77, -}; - -static const unsigned char chroma_ac_value[162 + 2] = { + /* values */ 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, @@ -124,6 +145,38 @@ static unsigned char chroma_q[64] = { 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, }; +static const unsigned char width_align[] = { + [CODA9_JPEG_FORMAT_420] = 16, + [CODA9_JPEG_FORMAT_422] = 16, + [CODA9_JPEG_FORMAT_224] = 8, + [CODA9_JPEG_FORMAT_444] = 8, + [CODA9_JPEG_FORMAT_400] = 8, +}; + +static const unsigned char height_align[] = { + [CODA9_JPEG_FORMAT_420] = 16, + [CODA9_JPEG_FORMAT_422] = 8, + [CODA9_JPEG_FORMAT_224] = 16, + [CODA9_JPEG_FORMAT_444] = 8, + [CODA9_JPEG_FORMAT_400] = 8, +}; + +static int coda9_jpeg_chroma_format(u32 pixfmt) +{ + switch (pixfmt) { + case V4L2_PIX_FMT_YUV420: + case V4L2_PIX_FMT_NV12: + return CODA9_JPEG_FORMAT_420; + case V4L2_PIX_FMT_YUV422P: + return CODA9_JPEG_FORMAT_422; + case V4L2_PIX_FMT_YUV444: + return CODA9_JPEG_FORMAT_444; + case V4L2_PIX_FMT_GREY: + return CODA9_JPEG_FORMAT_400; + } + return -EINVAL; +} + struct coda_memcpy_desc { int offset; const void *src; @@ -148,14 +201,10 @@ int coda_jpeg_write_tables(struct coda_ctx *ctx) { int i; static const struct coda_memcpy_desc huff[8] = { - { 0, luma_dc_bits, sizeof(luma_dc_bits) }, - { 16, luma_dc_value, sizeof(luma_dc_value) }, - { 32, luma_ac_bits, sizeof(luma_ac_bits) }, - { 48, luma_ac_value, sizeof(luma_ac_value) }, - { 216, chroma_dc_bits, sizeof(chroma_dc_bits) }, - { 232, chroma_dc_value, sizeof(chroma_dc_value) }, - { 248, chroma_ac_bits, sizeof(chroma_ac_bits) }, - { 264, chroma_ac_value, sizeof(chroma_ac_value) }, + { 0, luma_dc, sizeof(luma_dc) }, + { 32, luma_ac, sizeof(luma_ac) }, + { 216, chroma_dc, sizeof(chroma_dc) }, + { 248, chroma_ac, sizeof(chroma_ac) }, }; struct coda_memcpy_desc qmat[3] = { { 512, ctx->params.jpeg_qmat_tab[0], 64 }, @@ -198,6 +247,379 @@ bool coda_jpeg_check_buffer(struct coda_ctx *ctx, struct vb2_buffer *vb) return false; } +static const int bus_req_num[] = { + [CODA9_JPEG_FORMAT_420] = 2, + [CODA9_JPEG_FORMAT_422] = 3, + [CODA9_JPEG_FORMAT_224] = 3, + [CODA9_JPEG_FORMAT_444] = 4, + [CODA9_JPEG_FORMAT_400] = 4, +}; + +#define MCU_INFO(mcu_block_num, comp_num, comp0_info, comp1_info, comp2_info) \ + (((mcu_block_num) << CODA9_JPEG_MCU_BLOCK_NUM_OFFSET) | \ + ((comp_num) << CODA9_JPEG_COMP_NUM_OFFSET) | \ + ((comp0_info) << CODA9_JPEG_COMP0_INFO_OFFSET) | \ + ((comp1_info) << CODA9_JPEG_COMP1_INFO_OFFSET) | \ + ((comp2_info) << CODA9_JPEG_COMP2_INFO_OFFSET)) + +static const u32 mcu_info[] = { + [CODA9_JPEG_FORMAT_420] = MCU_INFO(6, 3, 10, 5, 5), + [CODA9_JPEG_FORMAT_422] = MCU_INFO(4, 3, 9, 5, 5), + [CODA9_JPEG_FORMAT_224] = MCU_INFO(4, 3, 6, 5, 5), + [CODA9_JPEG_FORMAT_444] = MCU_INFO(3, 3, 5, 5, 5), + [CODA9_JPEG_FORMAT_400] = MCU_INFO(1, 1, 5, 0, 0), +}; + +/* + * Convert Huffman table specifcations to tables of codes and code lengths. + * For reference, see JPEG ITU-T.81 (ISO/IEC 10918-1) [1] + * + * [1] https://www.w3.org/Graphics/JPEG/itu-t81.pdf + */ +static int coda9_jpeg_gen_enc_huff_tab(struct coda_ctx *ctx, int tab_num, + int *ehufsi, int *ehufco) +{ + int i, j, k, lastk, si, code, maxsymbol; + const u8 *bits, *huffval; + struct { + int size[256]; + int code[256]; + } *huff; + static const unsigned char *huff_tabs[4] = { + luma_dc, luma_ac, chroma_dc, chroma_ac, + }; + int ret = -EINVAL; + + huff = kzalloc(sizeof(*huff), GFP_KERNEL); + if (!huff) + return -ENOMEM; + + bits = huff_tabs[tab_num]; + huffval = huff_tabs[tab_num] + 16; + + maxsymbol = tab_num & 1 ? 256 : 16; + + /* Figure C.1 - Generation of table of Huffman code sizes */ + k = 0; + for (i = 1; i <= 16; i++) { + j = bits[i - 1]; + if (k + j > maxsymbol) + goto out; + while (j--) + huff->size[k++] = i; + } + lastk = k; + + /* Figure C.2 - Generation of table of Huffman codes */ + k = 0; + code = 0; + si = huff->size[0]; + while (k < lastk) { + while (huff->size[k] == si) { + huff->code[k++] = code; + code++; + } + if (code >= (1 << si)) + goto out; + code <<= 1; + si++; + } + + /* Figure C.3 - Ordering procedure for encoding procedure code tables */ + for (k = 0; k < lastk; k++) { + i = huffval[k]; + if (i >= maxsymbol || ehufsi[i]) + goto out; + ehufco[i] = huff->code[k]; + ehufsi[i] = huff->size[k]; + } + + ret = 0; +out: + kfree(huff); + return ret; +} + +#define DC_TABLE_INDEX0 0 +#define AC_TABLE_INDEX0 1 +#define DC_TABLE_INDEX1 2 +#define AC_TABLE_INDEX1 3 + +static int coda9_jpeg_load_huff_tab(struct coda_ctx *ctx) +{ + struct { + int size[4][256]; + int code[4][256]; + } *huff; + u32 *huff_data; + int i, j; + int ret; + + huff = kzalloc(sizeof(*huff), GFP_KERNEL); + if (!huff) + return -ENOMEM; + + /* Generate all four (luma/chroma DC/AC) code/size lookup tables */ + for (i = 0; i < 4; i++) { + ret = coda9_jpeg_gen_enc_huff_tab(ctx, i, huff->size[i], + huff->code[i]); + if (ret) + goto out; + } + + if (!ctx->params.jpeg_huff_data) { + ctx->params.jpeg_huff_data = + kzalloc(sizeof(u32) * CODA9_JPEG_ENC_HUFF_DATA_SIZE, + GFP_KERNEL); + if (!ctx->params.jpeg_huff_data) { + ret = -ENOMEM; + goto out; + } + } + huff_data = ctx->params.jpeg_huff_data; + + for (j = 0; j < 4; j++) { + /* Store Huffman lookup tables in AC0, AC1, DC0, DC1 order */ + int t = (j == 0) ? AC_TABLE_INDEX0 : + (j == 1) ? AC_TABLE_INDEX1 : + (j == 2) ? DC_TABLE_INDEX0 : + DC_TABLE_INDEX1; + /* DC tables only have 16 entries */ + int len = (j < 2) ? 256 : 16; + + for (i = 0; i < len; i++) { + if (huff->size[t][i] == 0 && huff->code[t][i] == 0) + *(huff_data++) = 0; + else + *(huff_data++) = + ((huff->size[t][i] - 1) << 16) | + huff->code[t][i]; + } + } + + ret = 0; +out: + kfree(huff); + return ret; +} + +static void coda9_jpeg_write_huff_tab(struct coda_ctx *ctx) +{ + struct coda_dev *dev = ctx->dev; + u32 *huff_data = ctx->params.jpeg_huff_data; + int i; + + /* Write Huffman size/code lookup tables in AC0, AC1, DC0, DC1 order */ + coda_write(dev, 0x3, CODA9_REG_JPEG_HUFF_CTRL); + for (i = 0; i < CODA9_JPEG_ENC_HUFF_DATA_SIZE; i++) + coda_write(dev, *(huff_data++), CODA9_REG_JPEG_HUFF_DATA); + coda_write(dev, 0x0, CODA9_REG_JPEG_HUFF_CTRL); +} + +static inline void coda9_jpeg_write_qmat_quotients(struct coda_dev *dev, + u8 *qmat, int index) +{ + int i; + + coda_write(dev, index | 0x3, CODA9_REG_JPEG_QMAT_CTRL); + for (i = 0; i < 64; i++) + coda_write(dev, 0x80000 / qmat[i], CODA9_REG_JPEG_QMAT_DATA); + coda_write(dev, index, CODA9_REG_JPEG_QMAT_CTRL); +} + +static void coda9_jpeg_load_qmat_tab(struct coda_ctx *ctx) +{ + struct coda_dev *dev = ctx->dev; + u8 *luma_tab; + u8 *chroma_tab; + + luma_tab = ctx->params.jpeg_qmat_tab[0]; + if (!luma_tab) + luma_tab = luma_q; + + chroma_tab = ctx->params.jpeg_qmat_tab[1]; + if (!chroma_tab) + chroma_tab = chroma_q; + + coda9_jpeg_write_qmat_quotients(dev, luma_tab, 0x00); + coda9_jpeg_write_qmat_quotients(dev, chroma_tab, 0x40); + coda9_jpeg_write_qmat_quotients(dev, chroma_tab, 0x80); +} + +struct coda_jpeg_stream { + u8 *curr; + u8 *end; +}; + +static inline int coda_jpeg_put_byte(u8 byte, struct coda_jpeg_stream *stream) +{ + if (stream->curr >= stream->end) + return -EINVAL; + + *stream->curr++ = byte; + + return 0; +} + +static inline int coda_jpeg_put_word(u16 word, struct coda_jpeg_stream *stream) +{ + if (stream->curr + sizeof(__be16) > stream->end) + return -EINVAL; + + put_unaligned_be16(word, stream->curr); + stream->curr += sizeof(__be16); + + return 0; +} + +static int coda_jpeg_put_table(u16 marker, u8 index, const u8 *table, + size_t len, struct coda_jpeg_stream *stream) +{ + int i, ret; + + ret = coda_jpeg_put_word(marker, stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_word(3 + len, stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_byte(index, stream); + for (i = 0; i < len && ret == 0; i++) + ret = coda_jpeg_put_byte(table[i], stream); + + return ret; +} + +static int coda_jpeg_define_quantization_table(struct coda_ctx *ctx, u8 index, + struct coda_jpeg_stream *stream) +{ + return coda_jpeg_put_table(DQT_MARKER, index, + ctx->params.jpeg_qmat_tab[index], 64, + stream); +} + +static int coda_jpeg_define_huffman_table(u8 index, const u8 *table, size_t len, + struct coda_jpeg_stream *stream) +{ + return coda_jpeg_put_table(DHT_MARKER, index, table, len, stream); +} + +static int coda9_jpeg_encode_header(struct coda_ctx *ctx, int len, u8 *buf) +{ + struct coda_jpeg_stream stream = { buf, buf + len }; + struct coda_q_data *q_data_src; + int chroma_format, comp_num; + int i, ret, pad; + + q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT); + chroma_format = coda9_jpeg_chroma_format(q_data_src->fourcc); + if (chroma_format < 0) + return 0; + + /* Start Of Image */ + ret = coda_jpeg_put_word(SOI_MARKER, &stream); + if (ret < 0) + return ret; + + /* Define Restart Interval */ + if (ctx->params.jpeg_restart_interval) { + ret = coda_jpeg_put_word(DRI_MARKER, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_word(4, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_word(ctx->params.jpeg_restart_interval, + &stream); + if (ret < 0) + return ret; + } + + /* Define Quantization Tables */ + ret = coda_jpeg_define_quantization_table(ctx, 0x00, &stream); + if (ret < 0) + return ret; + if (chroma_format != CODA9_JPEG_FORMAT_400) { + ret = coda_jpeg_define_quantization_table(ctx, 0x01, &stream); + if (ret < 0) + return ret; + } + + /* Define Huffman Tables */ + ret = coda_jpeg_define_huffman_table(0x00, luma_dc, 16 + 12, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_define_huffman_table(0x10, luma_ac, 16 + 162, &stream); + if (ret < 0) + return ret; + if (chroma_format != CODA9_JPEG_FORMAT_400) { + ret = coda_jpeg_define_huffman_table(0x01, chroma_dc, 16 + 12, + &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_define_huffman_table(0x11, chroma_ac, 16 + 162, + &stream); + if (ret < 0) + return ret; + } + + /* Start Of Frame */ + ret = coda_jpeg_put_word(SOF_MARKER, &stream); + if (ret < 0) + return ret; + comp_num = (chroma_format == CODA9_JPEG_FORMAT_400) ? 1 : 3; + ret = coda_jpeg_put_word(8 + comp_num * 3, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_byte(0x08, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_word(q_data_src->height, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_word(q_data_src->width, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_byte(comp_num, &stream); + if (ret < 0) + return ret; + for (i = 0; i < comp_num; i++) { + static unsigned char subsampling[5][3] = { + [CODA9_JPEG_FORMAT_420] = { 0x22, 0x11, 0x11 }, + [CODA9_JPEG_FORMAT_422] = { 0x21, 0x11, 0x11 }, + [CODA9_JPEG_FORMAT_224] = { 0x12, 0x11, 0x11 }, + [CODA9_JPEG_FORMAT_444] = { 0x11, 0x11, 0x11 }, + [CODA9_JPEG_FORMAT_400] = { 0x11 }, + }; + + /* Component identifier, matches SOS */ + ret = coda_jpeg_put_byte(i + 1, &stream); + if (ret < 0) + return ret; + ret = coda_jpeg_put_byte(subsampling[chroma_format][i], + &stream); + if (ret < 0) + return ret; + /* Chroma table index */ + ret = coda_jpeg_put_byte((i == 0) ? 0 : 1, &stream); + if (ret < 0) + return ret; + } + + /* Pad to multiple of 8 bytes */ + pad = (stream.curr - buf) % 8; + if (pad) { + pad = 8 - pad; + while (pad--) { + ret = coda_jpeg_put_byte(0x00, &stream); + if (ret < 0) + return ret; + } + } + + return stream.curr - buf; +} + /* * Scale quantization table using nonlinear scaling factor * u8 qtab[64], scale [50,190] @@ -247,3 +669,279 @@ void coda_set_jpeg_compression_quality(struct coda_ctx *ctx, int quality) coda_scale_quant_table(ctx->params.jpeg_qmat_tab[1], scale); } } + +/* + * Encoder context operations + */ + +static int coda9_jpeg_start_encoding(struct coda_ctx *ctx) +{ + struct coda_dev *dev = ctx->dev; + int ret; + + ret = coda9_jpeg_load_huff_tab(ctx); + if (ret < 0) { + v4l2_err(&dev->v4l2_dev, "error loading Huffman tables\n"); + return ret; + } + if (!ctx->params.jpeg_qmat_tab[0]) + ctx->params.jpeg_qmat_tab[0] = kmalloc(64, GFP_KERNEL); + if (!ctx->params.jpeg_qmat_tab[1]) + ctx->params.jpeg_qmat_tab[1] = kmalloc(64, GFP_KERNEL); + coda_set_jpeg_compression_quality(ctx, ctx->params.jpeg_quality); + + return 0; +} + +static int coda9_jpeg_prepare_encode(struct coda_ctx *ctx) +{ + struct coda_q_data *q_data_src; + struct vb2_v4l2_buffer *src_buf, *dst_buf; + struct coda_dev *dev = ctx->dev; + u32 start_addr, end_addr; + u16 aligned_width, aligned_height; + bool chroma_interleave; + int chroma_format; + int header_len; + int ret; + ktime_t timeout; + + src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + q_data_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT); + + if (vb2_get_plane_payload(&src_buf->vb2_buf, 0) == 0) + vb2_set_plane_payload(&src_buf->vb2_buf, 0, + vb2_plane_size(&src_buf->vb2_buf, 0)); + + src_buf->sequence = ctx->osequence; + dst_buf->sequence = ctx->osequence; + ctx->osequence++; + + src_buf->flags |= V4L2_BUF_FLAG_KEYFRAME; + src_buf->flags &= ~V4L2_BUF_FLAG_PFRAME; + + coda_set_gdi_regs(ctx); + + start_addr = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0); + end_addr = start_addr + vb2_plane_size(&dst_buf->vb2_buf, 0); + + chroma_format = coda9_jpeg_chroma_format(q_data_src->fourcc); + if (chroma_format < 0) + return chroma_format; + + /* Round image dimensions to multiple of MCU size */ + aligned_width = round_up(q_data_src->width, width_align[chroma_format]); + aligned_height = round_up(q_data_src->height, + height_align[chroma_format]); + if (aligned_width != q_data_src->bytesperline) { + v4l2_err(&dev->v4l2_dev, "wrong stride: %d instead of %d\n", + aligned_width, q_data_src->bytesperline); + } + + header_len = + coda9_jpeg_encode_header(ctx, + vb2_plane_size(&dst_buf->vb2_buf, 0), + vb2_plane_vaddr(&dst_buf->vb2_buf, 0)); + if (header_len < 0) + return header_len; + + coda_write(dev, start_addr + header_len, CODA9_REG_JPEG_BBC_BAS_ADDR); + coda_write(dev, end_addr, CODA9_REG_JPEG_BBC_END_ADDR); + coda_write(dev, start_addr + header_len, CODA9_REG_JPEG_BBC_WR_PTR); + coda_write(dev, start_addr + header_len, CODA9_REG_JPEG_BBC_RD_PTR); + coda_write(dev, 0, CODA9_REG_JPEG_BBC_CUR_POS); + /* 64 words per 256-byte page */ + coda_write(dev, 64, CODA9_REG_JPEG_BBC_DATA_CNT); + coda_write(dev, start_addr, CODA9_REG_JPEG_BBC_EXT_ADDR); + coda_write(dev, 0, CODA9_REG_JPEG_BBC_INT_ADDR); + + coda_write(dev, 0, CODA9_REG_JPEG_GBU_BT_PTR); + coda_write(dev, 0, CODA9_REG_JPEG_GBU_WD_PTR); + coda_write(dev, 0, CODA9_REG_JPEG_GBU_BBSR); + coda_write(dev, 0, CODA9_REG_JPEG_BBC_STRM_CTRL); + coda_write(dev, 0, CODA9_REG_JPEG_GBU_CTRL); + coda_write(dev, 0, CODA9_REG_JPEG_GBU_FF_RPTR); + coda_write(dev, 127, CODA9_REG_JPEG_GBU_BBER); + coda_write(dev, 64, CODA9_REG_JPEG_GBU_BBIR); + coda_write(dev, 64, CODA9_REG_JPEG_GBU_BBHR); + + chroma_interleave = (q_data_src->fourcc == V4L2_PIX_FMT_NV12); + coda_write(dev, CODA9_JPEG_PIC_CTRL_TC_DIRECTION | + CODA9_JPEG_PIC_CTRL_ENCODER_EN, CODA9_REG_JPEG_PIC_CTRL); + coda_write(dev, 0, CODA9_REG_JPEG_SCL_INFO); + coda_write(dev, chroma_interleave, CODA9_REG_JPEG_DPB_CONFIG); + coda_write(dev, ctx->params.jpeg_restart_interval, + CODA9_REG_JPEG_RST_INTVAL); + coda_write(dev, 1, CODA9_REG_JPEG_BBC_CTRL); + + coda_write(dev, bus_req_num[chroma_format], CODA9_REG_JPEG_OP_INFO); + + coda9_jpeg_write_huff_tab(ctx); + coda9_jpeg_load_qmat_tab(ctx); + + if (ctx->params.rot_mode & CODA_ROT_90) { + aligned_width = aligned_height; + aligned_height = q_data_src->bytesperline; + if (chroma_format == CODA9_JPEG_FORMAT_422) + chroma_format = CODA9_JPEG_FORMAT_224; + else if (chroma_format == CODA9_JPEG_FORMAT_224) + chroma_format = CODA9_JPEG_FORMAT_422; + } + /* These need to be multiples of MCU size */ + coda_write(dev, aligned_width << 16 | aligned_height, + CODA9_REG_JPEG_PIC_SIZE); + coda_write(dev, ctx->params.rot_mode ? + (CODA_ROT_MIR_ENABLE | ctx->params.rot_mode) : 0, + CODA9_REG_JPEG_ROT_INFO); + + coda_write(dev, mcu_info[chroma_format], CODA9_REG_JPEG_MCU_INFO); + + coda_write(dev, 1, CODA9_GDI_CONTROL); + timeout = ktime_add_us(ktime_get(), 100000); + do { + ret = coda_read(dev, CODA9_GDI_STATUS); + if (ktime_compare(ktime_get(), timeout) > 0) { + v4l2_err(&dev->v4l2_dev, "timeout waiting for GDI\n"); + return -ETIMEDOUT; + } + } while (!ret); + + coda_write(dev, (chroma_format << 17) | (chroma_interleave << 16) | + q_data_src->bytesperline, CODA9_GDI_INFO_CONTROL); + /* The content of this register seems to be irrelevant: */ + coda_write(dev, aligned_width << 16 | aligned_height, + CODA9_GDI_INFO_PIC_SIZE); + + coda_write_base(ctx, q_data_src, src_buf, CODA9_GDI_INFO_BASE_Y); + + coda_write(dev, 0, CODA9_REG_JPEG_DPB_BASE00); + coda_write(dev, 0, CODA9_GDI_CONTROL); + coda_write(dev, 1, CODA9_GDI_PIC_INIT_HOST); + + coda_write(dev, 1, CODA9_GDI_WPROT_ERR_CLR); + coda_write(dev, 0, CODA9_GDI_WPROT_RGN_EN); + + trace_coda_jpeg_run(ctx, src_buf); + + coda_write(dev, 1, CODA9_REG_JPEG_PIC_START); + + return 0; +} + +static void coda9_jpeg_finish_encode(struct coda_ctx *ctx) +{ + struct vb2_v4l2_buffer *src_buf, *dst_buf; + struct coda_dev *dev = ctx->dev; + u32 wr_ptr, start_ptr; + u32 err_mb; + + if (ctx->aborting) { + coda_write(ctx->dev, 0, CODA9_REG_JPEG_BBC_FLUSH_CMD); + return; + } + + /* + * Lock to make sure that an encoder stop command running in parallel + * will either already have marked src_buf as last, or it will wake up + * the capture queue after the buffers are returned. + */ + mutex_lock(&ctx->wakeup_mutex); + src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + + trace_coda_jpeg_done(ctx, dst_buf); + + /* + * Set plane payload to the number of bytes written out + * by the JPEG processing unit + */ + start_ptr = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0); + wr_ptr = coda_read(dev, CODA9_REG_JPEG_BBC_WR_PTR); + vb2_set_plane_payload(&dst_buf->vb2_buf, 0, wr_ptr - start_ptr); + + err_mb = coda_read(dev, CODA9_REG_JPEG_PIC_ERRMB); + if (err_mb) + coda_dbg(1, ctx, "ERRMB: 0x%x\n", err_mb); + + coda_write(dev, 0, CODA9_REG_JPEG_BBC_FLUSH_CMD); + + dst_buf->flags &= ~(V4L2_BUF_FLAG_PFRAME | V4L2_BUF_FLAG_LAST); + dst_buf->flags |= V4L2_BUF_FLAG_KEYFRAME; + dst_buf->flags |= src_buf->flags & V4L2_BUF_FLAG_LAST; + + v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, false); + + v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE); + coda_m2m_buf_done(ctx, dst_buf, err_mb ? VB2_BUF_STATE_ERROR : + VB2_BUF_STATE_DONE); + mutex_unlock(&ctx->wakeup_mutex); + + coda_dbg(1, ctx, "job finished: encoded frame (%u)%s\n", + dst_buf->sequence, + (dst_buf->flags & V4L2_BUF_FLAG_LAST) ? " (last)" : ""); +} + +static void coda9_jpeg_release(struct coda_ctx *ctx) +{ + int i; + + if (ctx->params.jpeg_qmat_tab[0] == luma_q) + ctx->params.jpeg_qmat_tab[0] = NULL; + if (ctx->params.jpeg_qmat_tab[1] == chroma_q) + ctx->params.jpeg_qmat_tab[1] = NULL; + for (i = 0; i < 3; i++) + kfree(ctx->params.jpeg_qmat_tab[i]); + kfree(ctx->params.jpeg_huff_data); +} + +const struct coda_context_ops coda9_jpeg_encode_ops = { + .queue_init = coda_encoder_queue_init, + .start_streaming = coda9_jpeg_start_encoding, + .prepare_run = coda9_jpeg_prepare_encode, + .finish_run = coda9_jpeg_finish_encode, + .release = coda9_jpeg_release, +}; + +irqreturn_t coda9_jpeg_irq_handler(int irq, void *data) +{ + struct coda_dev *dev = data; + struct coda_ctx *ctx; + int status; + int err_mb; + + status = coda_read(dev, CODA9_REG_JPEG_PIC_STATUS); + if (status == 0) + return IRQ_HANDLED; + coda_write(dev, status, CODA9_REG_JPEG_PIC_STATUS); + + if (status & CODA9_JPEG_STATUS_OVERFLOW) + v4l2_err(&dev->v4l2_dev, "JPEG overflow\n"); + + if (status & CODA9_JPEG_STATUS_BBC_INT) + v4l2_err(&dev->v4l2_dev, "JPEG BBC interrupt\n"); + + if (status & CODA9_JPEG_STATUS_ERROR) { + v4l2_err(&dev->v4l2_dev, "JPEG error\n"); + + err_mb = coda_read(dev, CODA9_REG_JPEG_PIC_ERRMB); + if (err_mb) { + v4l2_err(&dev->v4l2_dev, + "ERRMB: 0x%x: rst idx %d, mcu pos (%d,%d)\n", + err_mb, err_mb >> 24, (err_mb >> 12) & 0xfff, + err_mb & 0xfff); + } + } + + ctx = v4l2_m2m_get_curr_priv(dev->m2m_dev); + if (!ctx) { + v4l2_err(&dev->v4l2_dev, + "Instance released before the end of transaction\n"); + mutex_unlock(&dev->coda_mutex); + return IRQ_HANDLED; + } + + complete(&ctx->completion); + + return IRQ_HANDLED; +} diff --git a/drivers/media/platform/coda/coda.h b/drivers/media/platform/coda/coda.h index 9f226140b486..43bda175f517 100644 --- a/drivers/media/platform/coda/coda.h +++ b/drivers/media/platform/coda/coda.h @@ -126,6 +126,7 @@ struct coda_params { u8 jpeg_quality; u8 jpeg_restart_interval; u8 *jpeg_qmat_tab[3]; + u32 *jpeg_huff_data; int codec_mode; int codec_mode_aux; enum v4l2_mpeg_video_multi_slice_mode slice_mode; @@ -366,7 +367,9 @@ void coda_set_jpeg_compression_quality(struct coda_ctx *ctx, int quality); extern const struct coda_context_ops coda_bit_encode_ops; extern const struct coda_context_ops coda_bit_decode_ops; +extern const struct coda_context_ops coda9_jpeg_encode_ops; irqreturn_t coda_irq_handler(int irq, void *data); +irqreturn_t coda9_jpeg_irq_handler(int irq, void *data); #endif /* __CODA_H__ */ diff --git a/drivers/media/platform/coda/coda_regs.h b/drivers/media/platform/coda/coda_regs.h index b17464b56d3d..da5bb3212528 100644 --- a/drivers/media/platform/coda/coda_regs.h +++ b/drivers/media/platform/coda/coda_regs.h @@ -451,12 +451,21 @@ #define CODA9_CMD_FIRMWARE_CODE_REV 0x1c4 #define CODA9_GDMA_BASE 0x1000 +#define CODA9_GDI_CONTROL (CODA9_GDMA_BASE + 0x034) +#define CODA9_GDI_PIC_INIT_HOST (CODA9_GDMA_BASE + 0x038) +#define CODA9_GDI_STATUS (CODA9_GDMA_BASE + 0x080) #define CODA9_GDI_WPROT_ERR_CLR (CODA9_GDMA_BASE + 0x0a0) #define CODA9_GDI_WPROT_RGN_EN (CODA9_GDMA_BASE + 0x0ac) #define CODA9_GDI_BUS_CTRL (CODA9_GDMA_BASE + 0x0f0) #define CODA9_GDI_BUS_STATUS (CODA9_GDMA_BASE + 0x0f4) +#define CODA9_GDI_INFO_CONTROL (CODA9_GDMA_BASE + 0x400) +#define CODA9_GDI_INFO_PIC_SIZE (CODA9_GDMA_BASE + 0x404) +#define CODA9_GDI_INFO_BASE_Y (CODA9_GDMA_BASE + 0x408) +#define CODA9_GDI_INFO_BASE_CB (CODA9_GDMA_BASE + 0x40c) +#define CODA9_GDI_INFO_BASE_CR (CODA9_GDMA_BASE + 0x410) + #define CODA9_GDI_XY2_CAS_0 (CODA9_GDMA_BASE + 0x800) #define CODA9_GDI_XY2_CAS_F (CODA9_GDMA_BASE + 0x83c) @@ -477,4 +486,78 @@ #define CODA9_GDI_RBC2_AXI_1F (CODA9_GDMA_BASE + 0x91c) #define CODA9_GDI_TILEDBUF_BASE (CODA9_GDMA_BASE + 0x920) +#define CODA9_JPEG_BASE 0x3000 +#define CODA9_REG_JPEG_PIC_START (CODA9_JPEG_BASE + 0x000) +#define CODA9_REG_JPEG_PIC_STATUS (CODA9_JPEG_BASE + 0x004) +#define CODA9_JPEG_STATUS_OVERFLOW BIT(3) +#define CODA9_JPEG_STATUS_BBC_INT BIT(2) +#define CODA9_JPEG_STATUS_ERROR BIT(1) +#define CODA9_JPEG_STATUS_DONE BIT(0) +#define CODA9_REG_JPEG_PIC_ERRMB (CODA9_JPEG_BASE + 0x008) +#define CODA9_JPEG_ERRMB_RESTART_IDX_MASK (0xf << 24) +#define CODA9_JPEG_ERRMB_MCU_POS_X_MASK (0xfff << 12) +#define CODA9_JPEG_ERRMB_MCU_POS_Y_MASK 0xfff +#define CODA9_REG_JPEG_PIC_CTRL (CODA9_JPEG_BASE + 0x010) +#define CODA9_JPEG_PIC_CTRL_USER_HUFFMAN_EN BIT(6) +#define CODA9_JPEG_PIC_CTRL_TC_DIRECTION BIT(4) +#define CODA9_JPEG_PIC_CTRL_ENCODER_EN BIT(3) +#define CODA9_REG_JPEG_PIC_SIZE (CODA9_JPEG_BASE + 0x014) +#define CODA9_REG_JPEG_MCU_INFO (CODA9_JPEG_BASE + 0x018) +#define CODA9_JPEG_MCU_BLOCK_NUM_OFFSET 16 +#define CODA9_JPEG_COMP_NUM_OFFSET 12 +#define CODA9_JPEG_COMP0_INFO_OFFSET 8 +#define CODA9_JPEG_COMP1_INFO_OFFSET 4 +#define CODA9_JPEG_COMP2_INFO_OFFSET 0 +#define CODA9_REG_JPEG_ROT_INFO (CODA9_JPEG_BASE + 0x01c) +#define CODA9_JPEG_ROT_MIR_ENABLE BIT(4) +#define CODA9_JPEG_ROT_MIR_MODE_MASK 0xf +#define CODA9_REG_JPEG_SCL_INFO (CODA9_JPEG_BASE + 0x020) +#define CODA9_JPEG_SCL_ENABLE BIT(4) +#define CODA9_JPEG_SCL_HOR_MODE_MASK (0x3 << 2) +#define CODA9_JPEG_SCL_VER_MODE_MASK (0x3 << 0) +#define CODA9_REG_JPEG_IF_INFO (CODA9_JPEG_BASE + 0x024) +#define CODA9_JPEG_SENS_IF_CLR BIT(1) +#define CODA9_JPEG_DISP_IF_CLR BIT(0) +#define CODA9_REG_JPEG_OP_INFO (CODA9_JPEG_BASE + 0x02c) +#define CODA9_JPEG_BUS_REQ_NUM_OFFSET 0 +#define CODA9_JPEG_BUS_REQ_NUM_MASK 0x7 +#define CODA9_REG_JPEG_DPB_CONFIG (CODA9_JPEG_BASE + 0x030) +#define CODA9_REG_JPEG_DPB_BASE00 (CODA9_JPEG_BASE + 0x040) +#define CODA9_REG_JPEG_HUFF_CTRL (CODA9_JPEG_BASE + 0x080) +#define CODA9_REG_JPEG_HUFF_ADDR (CODA9_JPEG_BASE + 0x084) +#define CODA9_REG_JPEG_HUFF_DATA (CODA9_JPEG_BASE + 0x088) +#define CODA9_REG_JPEG_QMAT_CTRL (CODA9_JPEG_BASE + 0x090) +#define CODA9_REG_JPEG_QMAT_ADDR (CODA9_JPEG_BASE + 0x094) +#define CODA9_REG_JPEG_QMAT_DATA (CODA9_JPEG_BASE + 0x098) +#define CODA9_REG_JPEG_RST_INTVAL (CODA9_JPEG_BASE + 0x0b0) +#define CODA9_REG_JPEG_RST_INDEX (CODA9_JPEG_BASE + 0x0b4) +#define CODA9_REG_JPEG_RST_COUNT (CODA9_JPEG_BASE + 0x0b8) +#define CODA9_REG_JPEG_DPCM_DIFF_Y (CODA9_JPEG_BASE + 0x0f0) +#define CODA9_REG_JPEG_DPCM_DIFF_CB (CODA9_JPEG_BASE + 0x0f4) +#define CODA9_REG_JPEG_DPCM_DIFF_CR (CODA9_JPEG_BASE + 0x0f8) +#define CODA9_REG_JPEG_GBU_CTRL (CODA9_JPEG_BASE + 0x100) +#define CODA9_REG_JPEG_GBU_BT_PTR (CODA9_JPEG_BASE + 0x110) +#define CODA9_REG_JPEG_GBU_WD_PTR (CODA9_JPEG_BASE + 0x114) +#define CODA9_REG_JPEG_GBU_TT_CNT (CODA9_JPEG_BASE + 0x118) +#define CODA9_REG_JPEG_GBU_BBSR (CODA9_JPEG_BASE + 0x140) +#define CODA9_REG_JPEG_GBU_BBER (CODA9_JPEG_BASE + 0x144) +#define CODA9_REG_JPEG_GBU_BBIR (CODA9_JPEG_BASE + 0x148) +#define CODA9_REG_JPEG_GBU_BBHR (CODA9_JPEG_BASE + 0x14c) +#define CODA9_REG_JPEG_GBU_BCNT (CODA9_JPEG_BASE + 0x158) +#define CODA9_REG_JPEG_GBU_FF_RPTR (CODA9_JPEG_BASE + 0x160) +#define CODA9_REG_JPEG_GBU_FF_WPTR (CODA9_JPEG_BASE + 0x164) +#define CODA9_REG_JPEG_BBC_END_ADDR (CODA9_JPEG_BASE + 0x208) +#define CODA9_REG_JPEG_BBC_WR_PTR (CODA9_JPEG_BASE + 0x20c) +#define CODA9_REG_JPEG_BBC_RD_PTR (CODA9_JPEG_BASE + 0x210) +#define CODA9_REG_JPEG_BBC_EXT_ADDR (CODA9_JPEG_BASE + 0x214) +#define CODA9_REG_JPEG_BBC_INT_ADDR (CODA9_JPEG_BASE + 0x218) +#define CODA9_REG_JPEG_BBC_DATA_CNT (CODA9_JPEG_BASE + 0x21c) +#define CODA9_REG_JPEG_BBC_COMMAND (CODA9_JPEG_BASE + 0x220) +#define CODA9_REG_JPEG_BBC_BUSY (CODA9_JPEG_BASE + 0x224) +#define CODA9_REG_JPEG_BBC_CTRL (CODA9_JPEG_BASE + 0x228) +#define CODA9_REG_JPEG_BBC_CUR_POS (CODA9_JPEG_BASE + 0x22c) +#define CODA9_REG_JPEG_BBC_BAS_ADDR (CODA9_JPEG_BASE + 0x230) +#define CODA9_REG_JPEG_BBC_STRM_CTRL (CODA9_JPEG_BASE + 0x234) +#define CODA9_REG_JPEG_BBC_FLUSH_CMD (CODA9_JPEG_BASE + 0x238) + #endif diff --git a/drivers/media/platform/coda/trace.h b/drivers/media/platform/coda/trace.h index 6cf58237fff2..c0791c847f7c 100644 --- a/drivers/media/platform/coda/trace.h +++ b/drivers/media/platform/coda/trace.h @@ -154,6 +154,16 @@ DEFINE_EVENT(coda_buf_meta_class, coda_dec_rot_done, TP_ARGS(ctx, buf, meta) ); +DEFINE_EVENT(coda_buf_class, coda_jpeg_run, + TP_PROTO(struct coda_ctx *ctx, struct vb2_v4l2_buffer *buf), + TP_ARGS(ctx, buf) +); + +DEFINE_EVENT(coda_buf_class, coda_jpeg_done, + TP_PROTO(struct coda_ctx *ctx, struct vb2_v4l2_buffer *buf), + TP_ARGS(ctx, buf) +); + #endif /* __CODA_TRACE_H__ */ #undef TRACE_INCLUDE_PATH diff --git a/drivers/media/platform/cros-ec-cec/cros-ec-cec.c b/drivers/media/platform/cros-ec-cec/cros-ec-cec.c index f048e8994785..0e7e2772f08f 100644 --- a/drivers/media/platform/cros-ec-cec/cros-ec-cec.c +++ b/drivers/media/platform/cros-ec-cec/cros-ec-cec.c @@ -14,7 +14,6 @@ #include <linux/cec.h> #include <linux/slab.h> #include <linux/interrupt.h> -#include <linux/mfd/cros_ec.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> #include <media/cec.h> diff --git a/drivers/media/platform/davinci/vpfe_capture.c b/drivers/media/platform/davinci/vpfe_capture.c index 916ed743d716..9b1d9643589b 100644 --- a/drivers/media/platform/davinci/vpfe_capture.c +++ b/drivers/media/platform/davinci/vpfe_capture.c @@ -168,21 +168,22 @@ int vpfe_register_ccdc_device(const struct ccdc_hw_device *dev) int ret = 0; printk(KERN_NOTICE "vpfe_register_ccdc_device: %s\n", dev->name); - BUG_ON(!dev->hw_ops.open); - BUG_ON(!dev->hw_ops.enable); - BUG_ON(!dev->hw_ops.set_hw_if_params); - BUG_ON(!dev->hw_ops.configure); - BUG_ON(!dev->hw_ops.set_buftype); - BUG_ON(!dev->hw_ops.get_buftype); - BUG_ON(!dev->hw_ops.enum_pix); - BUG_ON(!dev->hw_ops.set_frame_format); - BUG_ON(!dev->hw_ops.get_frame_format); - BUG_ON(!dev->hw_ops.get_pixel_format); - BUG_ON(!dev->hw_ops.set_pixel_format); - BUG_ON(!dev->hw_ops.set_image_window); - BUG_ON(!dev->hw_ops.get_image_window); - BUG_ON(!dev->hw_ops.get_line_length); - BUG_ON(!dev->hw_ops.getfid); + if (!dev->hw_ops.open || + !dev->hw_ops.enable || + !dev->hw_ops.set_hw_if_params || + !dev->hw_ops.configure || + !dev->hw_ops.set_buftype || + !dev->hw_ops.get_buftype || + !dev->hw_ops.enum_pix || + !dev->hw_ops.set_frame_format || + !dev->hw_ops.get_frame_format || + !dev->hw_ops.get_pixel_format || + !dev->hw_ops.set_pixel_format || + !dev->hw_ops.set_image_window || + !dev->hw_ops.get_image_window || + !dev->hw_ops.get_line_length || + !dev->hw_ops.getfid) + return -EINVAL; mutex_lock(&ccdc_lock); if (!ccdc_cfg) { diff --git a/drivers/media/platform/mtk-mdp/mtk_mdp_core.c b/drivers/media/platform/mtk-mdp/mtk_mdp_core.c index c1e29a46ae69..aeaed2cf4458 100644 --- a/drivers/media/platform/mtk-mdp/mtk_mdp_core.c +++ b/drivers/media/platform/mtk-mdp/mtk_mdp_core.c @@ -229,6 +229,9 @@ static int mtk_mdp_remove(struct platform_device *pdev) mtk_mdp_unregister_m2m_device(mdp); v4l2_device_unregister(&mdp->v4l2_dev); + flush_workqueue(mdp->wdt_wq); + destroy_workqueue(mdp->wdt_wq); + flush_workqueue(mdp->job_wq); destroy_workqueue(mdp->job_wq); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c index 858727824889..0f3e710aed4e 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c @@ -104,6 +104,7 @@ static struct vb2_buffer *get_display_buffer(struct mtk_vcodec_ctx *ctx) { struct vdec_fb *disp_frame_buffer = NULL; struct mtk_video_dec_buf *dstbuf; + struct vb2_v4l2_buffer *vb; mtk_v4l2_debug(3, "[%d]", ctx->id); if (vdec_if_get_param(ctx, @@ -121,25 +122,26 @@ static struct vb2_buffer *get_display_buffer(struct mtk_vcodec_ctx *ctx) dstbuf = container_of(disp_frame_buffer, struct mtk_video_dec_buf, frame_buffer); + vb = &dstbuf->m2m_buf.vb; mutex_lock(&ctx->lock); if (dstbuf->used) { - vb2_set_plane_payload(&dstbuf->vb.vb2_buf, 0, - ctx->picinfo.fb_sz[0]); + vb2_set_plane_payload(&vb->vb2_buf, 0, + ctx->picinfo.fb_sz[0]); if (ctx->q_data[MTK_Q_DATA_DST].fmt->num_planes == 2) - vb2_set_plane_payload(&dstbuf->vb.vb2_buf, 1, + vb2_set_plane_payload(&vb->vb2_buf, 1, ctx->picinfo.fb_sz[1]); mtk_v4l2_debug(2, "[%d]status=%x queue id=%d to done_list %d", ctx->id, disp_frame_buffer->status, - dstbuf->vb.vb2_buf.index, + vb->vb2_buf.index, dstbuf->queued_in_vb2); - v4l2_m2m_buf_done(&dstbuf->vb, VB2_BUF_STATE_DONE); + v4l2_m2m_buf_done(vb, VB2_BUF_STATE_DONE); ctx->decoded_frame_cnt++; } mutex_unlock(&ctx->lock); - return &dstbuf->vb.vb2_buf; + return &vb->vb2_buf; } /* @@ -154,6 +156,7 @@ static struct vb2_buffer *get_free_buffer(struct mtk_vcodec_ctx *ctx) { struct mtk_video_dec_buf *dstbuf; struct vdec_fb *free_frame_buffer = NULL; + struct vb2_v4l2_buffer *vb; if (vdec_if_get_param(ctx, GET_PARAM_FREE_FRAME_BUFFER, @@ -171,6 +174,7 @@ static struct vb2_buffer *get_free_buffer(struct mtk_vcodec_ctx *ctx) dstbuf = container_of(free_frame_buffer, struct mtk_video_dec_buf, frame_buffer); + vb = &dstbuf->m2m_buf.vb; mutex_lock(&ctx->lock); if (dstbuf->used) { @@ -187,9 +191,9 @@ static struct vb2_buffer *get_free_buffer(struct mtk_vcodec_ctx *ctx) mtk_v4l2_debug(2, "[%d]status=%x queue id=%d to rdy_queue %d", ctx->id, free_frame_buffer->status, - dstbuf->vb.vb2_buf.index, + vb->vb2_buf.index, dstbuf->queued_in_vb2); - v4l2_m2m_buf_queue(ctx->m2m_ctx, &dstbuf->vb); + v4l2_m2m_buf_queue(ctx->m2m_ctx, vb); } else if ((dstbuf->queued_in_vb2 == false) && (dstbuf->queued_in_v4l2 == true)) { /* @@ -205,8 +209,8 @@ static struct vb2_buffer *get_free_buffer(struct mtk_vcodec_ctx *ctx) mtk_v4l2_debug(2, "[%d]status=%x queue id=%d to rdy_queue", ctx->id, free_frame_buffer->status, - dstbuf->vb.vb2_buf.index); - v4l2_m2m_buf_queue(ctx->m2m_ctx, &dstbuf->vb); + vb->vb2_buf.index); + v4l2_m2m_buf_queue(ctx->m2m_ctx, vb); dstbuf->queued_in_vb2 = true; } else { /* @@ -219,14 +223,14 @@ static struct vb2_buffer *get_free_buffer(struct mtk_vcodec_ctx *ctx) */ mtk_v4l2_debug(3, "[%d]status=%x err queue id=%d %d %d", ctx->id, free_frame_buffer->status, - dstbuf->vb.vb2_buf.index, + vb->vb2_buf.index, dstbuf->queued_in_vb2, dstbuf->queued_in_v4l2); } dstbuf->used = false; } mutex_unlock(&ctx->lock); - return &dstbuf->vb.vb2_buf; + return &vb->vb2_buf; } static void clean_display_buffer(struct mtk_vcodec_ctx *ctx) @@ -365,8 +369,10 @@ static void mtk_vdec_worker(struct work_struct *work) return; } - src_buf_info = container_of(src_buf, struct mtk_video_dec_buf, vb); - dst_buf_info = container_of(dst_buf, struct mtk_video_dec_buf, vb); + src_buf_info = container_of(src_buf, struct mtk_video_dec_buf, + m2m_buf.vb); + dst_buf_info = container_of(dst_buf, struct mtk_video_dec_buf, + m2m_buf.vb); pfb = &dst_buf_info->frame_buffer; pfb->base_y.va = vb2_plane_vaddr(&dst_buf->vb2_buf, 0); @@ -397,11 +403,11 @@ static void mtk_vdec_worker(struct work_struct *work) vdec_if_decode(ctx, NULL, NULL, &res_chg); clean_display_buffer(ctx); - vb2_set_plane_payload(&dst_buf_info->vb.vb2_buf, 0, 0); + vb2_set_plane_payload(&dst_buf->vb2_buf, 0, 0); if (ctx->q_data[MTK_Q_DATA_DST].fmt->num_planes == 2) - vb2_set_plane_payload(&dst_buf_info->vb.vb2_buf, 1, 0); + vb2_set_plane_payload(&dst_buf->vb2_buf, 1, 0); dst_buf->flags |= V4L2_BUF_FLAG_LAST; - v4l2_m2m_buf_done(&dst_buf_info->vb, VB2_BUF_STATE_DONE); + v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_DONE); clean_free_buffer(ctx); v4l2_m2m_job_finish(dev->m2m_dev_dec, ctx->m2m_ctx); return; @@ -417,10 +423,8 @@ static void mtk_vdec_worker(struct work_struct *work) } mtk_v4l2_debug(3, "[%d] Bitstream VA=%p DMA=%pad Size=%zx vb=%p", ctx->id, buf.va, &buf.dma_addr, buf.size, src_buf); - dst_buf_info->vb.vb2_buf.timestamp - = src_buf_info->vb.vb2_buf.timestamp; - dst_buf_info->vb.timecode - = src_buf_info->vb.timecode; + dst_buf->vb2_buf.timestamp = src_buf->vb2_buf.timestamp; + dst_buf->timecode = src_buf->timecode; mutex_lock(&ctx->lock); dst_buf_info->used = true; mutex_unlock(&ctx->lock); @@ -434,7 +438,7 @@ static void mtk_vdec_worker(struct work_struct *work) ctx->id, src_buf->vb2_buf.index, buf.size, - src_buf_info->vb.vb2_buf.timestamp, + src_buf->vb2_buf.timestamp, dst_buf->vb2_buf.index, ret, res_chg); src_buf = v4l2_m2m_src_buf_remove(ctx->m2m_ctx); @@ -443,14 +447,14 @@ static void mtk_vdec_worker(struct work_struct *work) src_buf_info->error = true; mutex_unlock(&ctx->lock); } - v4l2_m2m_buf_done(&src_buf_info->vb, VB2_BUF_STATE_ERROR); + v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR); } else if (res_chg == false) { /* * we only return src buffer with VB2_BUF_STATE_DONE * when decode success without resolution change */ src_buf = v4l2_m2m_src_buf_remove(ctx->m2m_ctx); - v4l2_m2m_buf_done(&src_buf_info->vb, VB2_BUF_STATE_DONE); + v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE); } dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); @@ -522,7 +526,8 @@ static int vidioc_decoder_cmd(struct file *file, void *priv, mtk_v4l2_debug(1, "Capture stream is off. No need to flush."); return 0; } - v4l2_m2m_buf_queue(ctx->m2m_ctx, &ctx->empty_flush_buf->vb); + v4l2_m2m_buf_queue(ctx->m2m_ctx, + &ctx->empty_flush_buf->m2m_buf.vb); v4l2_m2m_try_schedule(ctx->m2m_ctx); break; @@ -1148,7 +1153,8 @@ static void vb2ops_vdec_buf_queue(struct vb2_buffer *vb) */ if (vb->vb2_queue->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { vb2_v4l2 = to_vb2_v4l2_buffer(vb); - buf = container_of(vb2_v4l2, struct mtk_video_dec_buf, vb); + buf = container_of(vb2_v4l2, struct mtk_video_dec_buf, + m2m_buf.vb); mutex_lock(&ctx->lock); if (buf->used == false) { v4l2_m2m_buf_queue(ctx->m2m_ctx, vb2_v4l2); @@ -1175,7 +1181,7 @@ static void vb2ops_vdec_buf_queue(struct vb2_buffer *vb) mtk_v4l2_err("No src buffer"); return; } - buf = container_of(src_buf, struct mtk_video_dec_buf, vb); + buf = container_of(src_buf, struct mtk_video_dec_buf, m2m_buf.vb); if (buf->lastframe) { /* This shouldn't happen. Just in case. */ mtk_v4l2_err("Invalid flush buffer."); @@ -1256,7 +1262,7 @@ static void vb2ops_vdec_buf_finish(struct vb2_buffer *vb) bool buf_error; vb2_v4l2 = container_of(vb, struct vb2_v4l2_buffer, vb2_buf); - buf = container_of(vb2_v4l2, struct mtk_video_dec_buf, vb); + buf = container_of(vb2_v4l2, struct mtk_video_dec_buf, m2m_buf.vb); mutex_lock(&ctx->lock); if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { buf->queued_in_v4l2 = false; @@ -1276,7 +1282,7 @@ static int vb2ops_vdec_buf_init(struct vb2_buffer *vb) struct vb2_v4l2_buffer *vb2_v4l2 = container_of(vb, struct vb2_v4l2_buffer, vb2_buf); struct mtk_video_dec_buf *buf = container_of(vb2_v4l2, - struct mtk_video_dec_buf, vb); + struct mtk_video_dec_buf, m2m_buf.vb); if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { buf->used = false; @@ -1309,7 +1315,7 @@ static void vb2ops_vdec_stop_streaming(struct vb2_queue *q) if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { while ((src_buf = v4l2_m2m_src_buf_remove(ctx->m2m_ctx))) { struct mtk_video_dec_buf *buf_info = container_of( - src_buf, struct mtk_video_dec_buf, vb); + src_buf, struct mtk_video_dec_buf, m2m_buf.vb); if (!buf_info->lastframe) v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h index e0c5338bde3d..cf26b6c1486a 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h @@ -9,7 +9,7 @@ #define _MTK_VCODEC_DEC_H_ #include <media/videobuf2-core.h> -#include <media/videobuf2-v4l2.h> +#include <media/v4l2-mem2mem.h> #define VCODEC_CAPABILITY_4K_DISABLED 0x10 #define VCODEC_DEC_4K_CODED_WIDTH 4096U @@ -33,7 +33,7 @@ struct vdec_fb { /** * struct mtk_video_dec_buf - Private data related to each VB2 buffer. - * @b: VB2 buffer + * @m2m_buf: M2M buffer * @list: link list * @used: Capture buffer contain decoded frame data and keep in * codec data structure @@ -47,8 +47,7 @@ struct vdec_fb { * Note : These status information help us track and debug buffer state */ struct mtk_video_dec_buf { - struct vb2_v4l2_buffer vb; - struct list_head list; + struct v4l2_m2m_buffer m2m_buf; bool used; bool queued_in_vb2; diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c index 944771ee5f5c..100ae8c5e702 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c @@ -137,7 +137,7 @@ static int fops_vcodec_open(struct file *file) } src_vq = v4l2_m2m_get_vq(ctx->m2m_ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); - ctx->empty_flush_buf->vb.vb2_buf.vb2_queue = src_vq; + ctx->empty_flush_buf->m2m_buf.vb.vb2_buf.vb2_queue = src_vq; ctx->empty_flush_buf->lastframe = true; mtk_vcodec_dec_set_default_params(ctx); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c index fd8de027e83e..d469ff6464b2 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c @@ -332,14 +332,12 @@ static int vidioc_try_fmt(struct v4l2_format *f, pix_fmt_mp->num_planes = fmt->num_planes; pix_fmt_mp->plane_fmt[0].sizeimage = - pix_fmt_mp->width * pix_fmt_mp->height + - ((ALIGN(pix_fmt_mp->width, 16) * 2) * 16); + pix_fmt_mp->width * pix_fmt_mp->height; pix_fmt_mp->plane_fmt[0].bytesperline = pix_fmt_mp->width; if (pix_fmt_mp->num_planes == 2) { pix_fmt_mp->plane_fmt[1].sizeimage = - (pix_fmt_mp->width * pix_fmt_mp->height) / 2 + - (ALIGN(pix_fmt_mp->width, 16) * 16); + (pix_fmt_mp->width * pix_fmt_mp->height) / 2; pix_fmt_mp->plane_fmt[2].sizeimage = 0; pix_fmt_mp->plane_fmt[1].bytesperline = pix_fmt_mp->width; @@ -347,8 +345,7 @@ static int vidioc_try_fmt(struct v4l2_format *f, } else if (pix_fmt_mp->num_planes == 3) { pix_fmt_mp->plane_fmt[1].sizeimage = pix_fmt_mp->plane_fmt[2].sizeimage = - (pix_fmt_mp->width * pix_fmt_mp->height) / 4 + - ((ALIGN(pix_fmt_mp->width, 16) / 2) * 16); + (pix_fmt_mp->width * pix_fmt_mp->height) / 4; pix_fmt_mp->plane_fmt[1].bytesperline = pix_fmt_mp->plane_fmt[2].bytesperline = pix_fmt_mp->width / 2; @@ -798,13 +795,14 @@ static void vb2ops_venc_buf_queue(struct vb2_buffer *vb) container_of(vb, struct vb2_v4l2_buffer, vb2_buf); struct mtk_video_enc_buf *mtk_buf = - container_of(vb2_v4l2, struct mtk_video_enc_buf, vb); + container_of(vb2_v4l2, struct mtk_video_enc_buf, + m2m_buf.vb); if ((vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) && (ctx->param_change != MTK_ENCODE_PARAM_NONE)) { mtk_v4l2_debug(1, "[%d] Before id=%d encode parameter change %x", ctx->id, - mtk_buf->vb.vb2_buf.index, + vb2_v4l2->vb2_buf.index, ctx->param_change); mtk_buf->param_change = ctx->param_change; mtk_buf->enc_params = ctx->enc_params; @@ -986,7 +984,8 @@ static int mtk_venc_param_change(struct mtk_vcodec_ctx *ctx) struct venc_enc_param enc_prm; struct vb2_v4l2_buffer *vb2_v4l2 = v4l2_m2m_next_src_buf(ctx->m2m_ctx); struct mtk_video_enc_buf *mtk_buf = - container_of(vb2_v4l2, struct mtk_video_enc_buf, vb); + container_of(vb2_v4l2, struct mtk_video_enc_buf, + m2m_buf.vb); int ret = 0; @@ -998,7 +997,7 @@ static int mtk_venc_param_change(struct mtk_vcodec_ctx *ctx) enc_prm.bitrate = mtk_buf->enc_params.bitrate; mtk_v4l2_debug(1, "[%d] id=%d, change param br=%d", ctx->id, - mtk_buf->vb.vb2_buf.index, + vb2_v4l2->vb2_buf.index, enc_prm.bitrate); ret |= venc_if_set_param(ctx, VENC_SET_PARAM_ADJUST_BITRATE, @@ -1009,7 +1008,7 @@ static int mtk_venc_param_change(struct mtk_vcodec_ctx *ctx) mtk_buf->enc_params.framerate_denom; mtk_v4l2_debug(1, "[%d] id=%d, change param fr=%d", ctx->id, - mtk_buf->vb.vb2_buf.index, + vb2_v4l2->vb2_buf.index, enc_prm.frm_rate); ret |= venc_if_set_param(ctx, VENC_SET_PARAM_ADJUST_FRAMERATE, @@ -1026,7 +1025,7 @@ static int mtk_venc_param_change(struct mtk_vcodec_ctx *ctx) if (!ret && mtk_buf->param_change & MTK_ENCODE_PARAM_FORCE_INTRA) { mtk_v4l2_debug(1, "[%d] id=%d, change param force I=%d", ctx->id, - mtk_buf->vb.vb2_buf.index, + vb2_v4l2->vb2_buf.index, mtk_buf->enc_params.force_intra); if (mtk_buf->enc_params.force_intra) ret |= venc_if_set_param(ctx, diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h index a9c9f86b9c83..513ee7993e34 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h @@ -9,7 +9,7 @@ #define _MTK_VCODEC_ENC_H_ #include <media/videobuf2-core.h> -#include <media/videobuf2-v4l2.h> +#include <media/v4l2-mem2mem.h> #define MTK_VENC_IRQ_STATUS_SPS 0x1 #define MTK_VENC_IRQ_STATUS_PPS 0x2 @@ -23,15 +23,15 @@ /** * struct mtk_video_enc_buf - Private data related to each VB2 buffer. - * @vb: Pointer to related VB2 buffer. + * @m2m_buf: M2M buffer * @list: list that buffer link to * @param_change: Types of encode parameter change before encoding this * buffer * @enc_params: Encode parameters changed before encode this buffer */ struct mtk_video_enc_buf { - struct vb2_v4l2_buffer vb; - struct list_head list; + struct v4l2_m2m_buffer m2m_buf; + u32 param_change; struct mtk_enc_params enc_params; }; diff --git a/drivers/media/platform/omap3isp/isp.c b/drivers/media/platform/omap3isp/isp.c index 327c5716922a..a4ee6b86663e 100644 --- a/drivers/media/platform/omap3isp/isp.c +++ b/drivers/media/platform/omap3isp/isp.c @@ -810,6 +810,10 @@ static int isp_pipeline_disable(struct isp_pipeline *pipe) ret = v4l2_subdev_call(subdev, video, s_stream, 0); + /* Stop at the first external sub-device. */ + if (subdev->dev != isp->dev) + break; + if (subdev == &isp->isp_res.subdev) ret |= isp_pipeline_wait(isp, isp_pipeline_wait_resizer); else if (subdev == &isp->isp_prev.subdev) @@ -837,10 +841,6 @@ static int isp_pipeline_disable(struct isp_pipeline *pipe) &subdev->entity); failure = -ETIMEDOUT; } - - /* Stop at the first external sub-device. */ - if (subdev->dev != isp->dev) - break; } return failure; diff --git a/drivers/media/platform/omap3isp/ispccdc.c b/drivers/media/platform/omap3isp/ispccdc.c index e2f336c715a4..471ae7cdb813 100644 --- a/drivers/media/platform/omap3isp/ispccdc.c +++ b/drivers/media/platform/omap3isp/ispccdc.c @@ -1607,6 +1607,11 @@ static int ccdc_isr_buffer(struct isp_ccdc_device *ccdc) return 0; } + /* Don't restart CCDC if we're just about to stop streaming. */ + if (ccdc->state == ISP_PIPELINE_STREAM_CONTINUOUS && + ccdc->stopping & CCDC_STOP_REQUEST) + return 0; + if (!ccdc_has_all_fields(ccdc)) return 1; @@ -1661,16 +1666,15 @@ static void ccdc_vd0_isr(struct isp_ccdc_device *ccdc) spin_unlock_irqrestore(&ccdc->lock, flags); } - if (ccdc->output & CCDC_OUTPUT_MEMORY) - restart = ccdc_isr_buffer(ccdc); - spin_lock_irqsave(&ccdc->lock, flags); - if (ccdc_handle_stopping(ccdc, CCDC_EVENT_VD0)) { spin_unlock_irqrestore(&ccdc->lock, flags); return; } + if (ccdc->output & CCDC_OUTPUT_MEMORY) + restart = ccdc_isr_buffer(ccdc); + if (!ccdc->shadow_update) ccdc_apply_controls(ccdc); spin_unlock_irqrestore(&ccdc->lock, flags); diff --git a/drivers/media/platform/pxa_camera.c b/drivers/media/platform/pxa_camera.c index 8d47ea0c33f8..43ae645d866b 100644 --- a/drivers/media/platform/pxa_camera.c +++ b/drivers/media/platform/pxa_camera.c @@ -2530,6 +2530,7 @@ exit_free_v4l2dev: v4l2_device_unregister(&pcdev->v4l2_dev); exit_deactivate: pxa_camera_deactivate(pcdev); + tasklet_kill(&pcdev->task_eof); exit_free_dma: dma_release_channel(pcdev->dma_chans[2]); exit_free_dma_u: @@ -2544,6 +2545,7 @@ static int pxa_camera_remove(struct platform_device *pdev) struct pxa_camera_dev *pcdev = dev_get_drvdata(&pdev->dev); pxa_camera_deactivate(pcdev); + tasklet_kill(&pcdev->task_eof); dma_release_channel(pcdev->dma_chans[0]); dma_release_channel(pcdev->dma_chans[1]); dma_release_channel(pcdev->dma_chans[2]); diff --git a/drivers/media/platform/rcar-vin/rcar-v4l2.c b/drivers/media/platform/rcar-vin/rcar-v4l2.c index 9e2e63ffcc47..5ff565e76bca 100644 --- a/drivers/media/platform/rcar-vin/rcar-v4l2.c +++ b/drivers/media/platform/rcar-vin/rcar-v4l2.c @@ -144,7 +144,7 @@ static void rvin_format_align(struct rvin_dev *vin, struct v4l2_pix_format *pix) } /* HW limit width to a multiple of 32 (2^5) for NV12/16 else 2 (2^1) */ - switch (vin->format.pixelformat) { + switch (pix->pixelformat) { case V4L2_PIX_FMT_NV12: case V4L2_PIX_FMT_NV16: walign = 5; diff --git a/drivers/media/platform/sti/bdisp/bdisp-hw.c b/drivers/media/platform/sti/bdisp/bdisp-hw.c index 4372abbb5950..a74e9fd65238 100644 --- a/drivers/media/platform/sti/bdisp/bdisp-hw.c +++ b/drivers/media/platform/sti/bdisp/bdisp-hw.c @@ -14,8 +14,8 @@ #define MAX_SRC_WIDTH 2048 /* Reset & boot poll config */ -#define POLL_RST_MAX 50 -#define POLL_RST_DELAY_MS 20 +#define POLL_RST_MAX 500 +#define POLL_RST_DELAY_MS 2 enum bdisp_target_plan { BDISP_RGB, @@ -382,7 +382,7 @@ int bdisp_hw_reset(struct bdisp_dev *bdisp) for (i = 0; i < POLL_RST_MAX; i++) { if (readl(bdisp->regs + BLT_STA1) & BLT_STA1_IDLE) break; - msleep(POLL_RST_DELAY_MS); + udelay(POLL_RST_DELAY_MS * 1000); } if (i == POLL_RST_MAX) dev_err(bdisp->dev, "Reset timeout\n"); diff --git a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c index 675b5f2b4c2e..d1025a53709f 100644 --- a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c +++ b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c @@ -1274,6 +1274,8 @@ static int bdisp_remove(struct platform_device *pdev) if (!IS_ERR(bdisp->clock)) clk_unprepare(bdisp->clock); + destroy_workqueue(bdisp->work_queue); + dev_dbg(&pdev->dev, "%s driver unloaded\n", pdev->name); return 0; @@ -1317,20 +1319,22 @@ static int bdisp_probe(struct platform_device *pdev) bdisp->regs = devm_ioremap_resource(dev, res); if (IS_ERR(bdisp->regs)) { dev_err(dev, "failed to get regs\n"); - return PTR_ERR(bdisp->regs); + ret = PTR_ERR(bdisp->regs); + goto err_wq; } bdisp->clock = devm_clk_get(dev, BDISP_NAME); if (IS_ERR(bdisp->clock)) { dev_err(dev, "failed to get clock\n"); - return PTR_ERR(bdisp->clock); + ret = PTR_ERR(bdisp->clock); + goto err_wq; } ret = clk_prepare(bdisp->clock); if (ret < 0) { dev_err(dev, "clock prepare failed\n"); bdisp->clock = ERR_PTR(-EINVAL); - return ret; + goto err_wq; } res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); @@ -1402,7 +1406,8 @@ err_v4l2: err_clk: if (!IS_ERR(bdisp->clock)) clk_unprepare(bdisp->clock); - +err_wq: + destroy_workqueue(bdisp->work_queue); return ret; } diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c index a79250a7f812..0560a9cb004b 100644 --- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c +++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c @@ -170,8 +170,9 @@ int c8sectpfe_frontend_attach(struct dvb_frontend **fe, /* attach tuner */ request_module("tda18212"); - client = i2c_new_device(tsin->i2c_adapter, &tda18212_info); - if (!client || !client->dev.driver) { + client = i2c_new_client_device(tsin->i2c_adapter, + &tda18212_info); + if (!i2c_client_has_driver(client)) { dvb_frontend_detach(*fe); return -ENODEV; } diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c index f36dc6258900..eff34ded6305 100644 --- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c +++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c @@ -11,6 +11,7 @@ #include <linux/module.h> #include <linux/mutex.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/of_graph.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -28,6 +29,12 @@ #include "sun4i_csi.h" +struct sun4i_csi_traits { + unsigned int channels; + unsigned int max_width; + bool has_isp; +}; + static const struct media_entity_operations sun4i_csi_video_entity_ops = { .link_validate = v4l2_subdev_link_validate, }; @@ -155,6 +162,31 @@ static int sun4i_csi_probe(struct platform_device *pdev) subdev = &csi->subdev; vdev = &csi->vdev; + csi->traits = of_device_get_match_data(&pdev->dev); + if (!csi->traits) + return -EINVAL; + + /* + * On Allwinner SoCs, some high memory bandwidth devices do DMA + * directly over the memory bus (called MBUS), instead of the + * system bus. The memory bus has a different addressing scheme + * without the DRAM starting offset. + * + * In some cases this can be described by an interconnect in + * the device tree. In other cases where the hardware is not + * fully understood and the interconnect is left out of the + * device tree, fall back to a default offset. + */ + if (of_find_property(csi->dev->of_node, "interconnects", NULL)) { + ret = of_dma_configure(csi->dev, csi->dev->of_node, true); + if (ret) + return ret; + } else { +#ifdef PHYS_PFN_OFFSET + csi->dev->dma_pfn_offset = PHYS_PFN_OFFSET; +#endif + } + csi->mdev.dev = csi->dev; strscpy(csi->mdev.model, "Allwinner Video Capture Device", sizeof(csi->mdev.model)); @@ -177,10 +209,12 @@ static int sun4i_csi_probe(struct platform_device *pdev) return PTR_ERR(csi->bus_clk); } - csi->isp_clk = devm_clk_get(&pdev->dev, "isp"); - if (IS_ERR(csi->isp_clk)) { - dev_err(&pdev->dev, "Couldn't get our ISP clock\n"); - return PTR_ERR(csi->isp_clk); + if (csi->traits->has_isp) { + csi->isp_clk = devm_clk_get(&pdev->dev, "isp"); + if (IS_ERR(csi->isp_clk)) { + dev_err(&pdev->dev, "Couldn't get our ISP clock\n"); + return PTR_ERR(csi->isp_clk); + } } csi->ram_clk = devm_clk_get(&pdev->dev, "ram"); @@ -258,8 +292,21 @@ static int sun4i_csi_remove(struct platform_device *pdev) return 0; } +static const struct sun4i_csi_traits sun4i_a10_csi1_traits = { + .channels = 1, + .max_width = 24, + .has_isp = false, +}; + +static const struct sun4i_csi_traits sun7i_a20_csi0_traits = { + .channels = 4, + .max_width = 16, + .has_isp = true, +}; + static const struct of_device_id sun4i_csi_of_match[] = { - { .compatible = "allwinner,sun7i-a20-csi0" }, + { .compatible = "allwinner,sun4i-a10-csi1", .data = &sun4i_a10_csi1_traits }, + { .compatible = "allwinner,sun7i-a20-csi0", .data = &sun7i_a20_csi0_traits }, { /* Sentinel */ } }; MODULE_DEVICE_TABLE(of, sun4i_csi_of_match); diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.h b/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.h index 001c8bde006c..0f67ff652c2e 100644 --- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.h +++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.h @@ -22,8 +22,8 @@ #define CSI_CFG_INPUT_FMT(fmt) ((fmt) << 20) #define CSI_CFG_OUTPUT_FMT(fmt) ((fmt) << 16) #define CSI_CFG_YUV_DATA_SEQ(seq) ((seq) << 8) -#define CSI_CFG_VSYNC_POL(pol) ((pol) << 2) -#define CSI_CFG_HSYNC_POL(pol) ((pol) << 1) +#define CSI_CFG_VREF_POL(pol) ((pol) << 2) +#define CSI_CFG_HREF_POL(pol) ((pol) << 1) #define CSI_CFG_PCLK_POL(pol) ((pol) << 0) #define CSI_CPT_CTRL_REG 0x08 @@ -108,6 +108,8 @@ struct sun4i_csi { /* Device resources */ struct device *dev; + const struct sun4i_csi_traits *traits; + void __iomem *regs; struct clk *bus_clk; struct clk *isp_clk; diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c index d6979e11a67b..78fa1c535ac6 100644 --- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c +++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c @@ -228,7 +228,7 @@ static int sun4i_csi_start_streaming(struct vb2_queue *vq, unsigned int count) struct sun4i_csi *csi = vb2_get_drv_priv(vq); struct v4l2_fwnode_bus_parallel *bus = &csi->bus; const struct sun4i_csi_format *csi_fmt; - unsigned long hsync_pol, pclk_pol, vsync_pol; + unsigned long href_pol, pclk_pol, vref_pol; unsigned long flags; unsigned int i; int ret; @@ -278,13 +278,21 @@ static int sun4i_csi_start_streaming(struct vb2_queue *vq, unsigned int count) writel(CSI_WIN_CTRL_H_ACTIVE(csi->fmt.height), csi->regs + CSI_WIN_CTRL_H_REG); - hsync_pol = !!(bus->flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH); - pclk_pol = !!(bus->flags & V4L2_MBUS_DATA_ACTIVE_HIGH); - vsync_pol = !!(bus->flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH); + /* + * This hardware uses [HV]REF instead of [HV]SYNC. Based on the + * provided timing diagrams in the manual, positive polarity + * equals active high [HV]REF. + * + * When the back porch is 0, [HV]REF is more or less equivalent + * to [HV]SYNC inverted. + */ + href_pol = !!(bus->flags & V4L2_MBUS_HSYNC_ACTIVE_LOW); + vref_pol = !!(bus->flags & V4L2_MBUS_VSYNC_ACTIVE_LOW); + pclk_pol = !!(bus->flags & V4L2_MBUS_PCLK_SAMPLE_RISING); writel(CSI_CFG_INPUT_FMT(csi_fmt->input) | CSI_CFG_OUTPUT_FMT(csi_fmt->output) | - CSI_CFG_VSYNC_POL(vsync_pol) | - CSI_CFG_HSYNC_POL(hsync_pol) | + CSI_CFG_VREF_POL(vref_pol) | + CSI_CFG_HREF_POL(href_pol) | CSI_CFG_PCLK_POL(pclk_pol), csi->regs + CSI_CFG_REG); diff --git a/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c b/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c index aaa1dc159ac2..b61f3dea7c93 100644 --- a/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c +++ b/drivers/media/platform/sunxi/sun8i-di/sun8i-di.c @@ -834,11 +834,8 @@ static int deinterlace_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); dev->base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(dev->base)) { - dev_err(dev->dev, "Failed to map registers\n"); - + if (IS_ERR(dev->base)) return PTR_ERR(dev->base); - } dev->bus_clk = devm_clk_get(dev->dev, "bus"); if (IS_ERR(dev->bus_clk)) { diff --git a/drivers/media/platform/ti-vpe/cal.c b/drivers/media/platform/ti-vpe/cal.c index 223161f9c403..be54806180a5 100644 --- a/drivers/media/platform/ti-vpe/cal.c +++ b/drivers/media/platform/ti-vpe/cal.c @@ -14,6 +14,8 @@ #include <linux/delay.h> #include <linux/pm_runtime.h> #include <linux/slab.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> #include <linux/videodev2.h> #include <linux/of_device.h> #include <linux/of_graph.h> @@ -32,8 +34,8 @@ #define CAL_MODULE_NAME "cal" -#define MAX_WIDTH 1920 -#define MAX_HEIGHT 1200 +#define MAX_WIDTH_BYTES (8192 * 8) +#define MAX_HEIGHT_LINES 16383 #define CAL_VERSION "0.1.0" @@ -71,8 +73,6 @@ static const struct v4l2_fract #define CAL_NUM_INPUT 1 #define CAL_NUM_CONTEXT 2 -#define bytes_per_line(pixel, bpp) (ALIGN(pixel * bpp, 16)) - #define reg_read(dev, offset) ioread32(dev->base + offset) #define reg_write(dev, offset, val) iowrite32(val, dev->base + offset) @@ -91,102 +91,103 @@ static const struct v4l2_fract struct cal_fmt { u32 fourcc; u32 code; - u8 depth; + /* Bits per pixel */ + u8 bpp; }; static struct cal_fmt cal_formats[] = { { .fourcc = V4L2_PIX_FMT_YUYV, .code = MEDIA_BUS_FMT_YUYV8_2X8, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_UYVY, .code = MEDIA_BUS_FMT_UYVY8_2X8, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_YVYU, .code = MEDIA_BUS_FMT_YVYU8_2X8, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_VYUY, .code = MEDIA_BUS_FMT_VYUY8_2X8, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_RGB565, /* gggbbbbb rrrrrggg */ .code = MEDIA_BUS_FMT_RGB565_2X8_LE, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_RGB565X, /* rrrrrggg gggbbbbb */ .code = MEDIA_BUS_FMT_RGB565_2X8_BE, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_RGB555, /* gggbbbbb arrrrrgg */ .code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_RGB555X, /* arrrrrgg gggbbbbb */ .code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE, - .depth = 16, + .bpp = 16, }, { .fourcc = V4L2_PIX_FMT_RGB24, /* rgb */ .code = MEDIA_BUS_FMT_RGB888_2X12_LE, - .depth = 24, + .bpp = 24, }, { .fourcc = V4L2_PIX_FMT_BGR24, /* bgr */ .code = MEDIA_BUS_FMT_RGB888_2X12_BE, - .depth = 24, + .bpp = 24, }, { .fourcc = V4L2_PIX_FMT_RGB32, /* argb */ .code = MEDIA_BUS_FMT_ARGB8888_1X32, - .depth = 32, + .bpp = 32, }, { .fourcc = V4L2_PIX_FMT_SBGGR8, .code = MEDIA_BUS_FMT_SBGGR8_1X8, - .depth = 8, + .bpp = 8, }, { .fourcc = V4L2_PIX_FMT_SGBRG8, .code = MEDIA_BUS_FMT_SGBRG8_1X8, - .depth = 8, + .bpp = 8, }, { .fourcc = V4L2_PIX_FMT_SGRBG8, .code = MEDIA_BUS_FMT_SGRBG8_1X8, - .depth = 8, + .bpp = 8, }, { .fourcc = V4L2_PIX_FMT_SRGGB8, .code = MEDIA_BUS_FMT_SRGGB8_1X8, - .depth = 8, + .bpp = 8, }, { .fourcc = V4L2_PIX_FMT_SBGGR10, .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .depth = 16, + .bpp = 10, }, { .fourcc = V4L2_PIX_FMT_SGBRG10, .code = MEDIA_BUS_FMT_SGBRG10_1X10, - .depth = 16, + .bpp = 10, }, { .fourcc = V4L2_PIX_FMT_SGRBG10, .code = MEDIA_BUS_FMT_SGRBG10_1X10, - .depth = 16, + .bpp = 10, }, { .fourcc = V4L2_PIX_FMT_SRGGB10, .code = MEDIA_BUS_FMT_SRGGB10_1X10, - .depth = 16, + .bpp = 10, }, { .fourcc = V4L2_PIX_FMT_SBGGR12, .code = MEDIA_BUS_FMT_SBGGR12_1X12, - .depth = 16, + .bpp = 12, }, { .fourcc = V4L2_PIX_FMT_SGBRG12, .code = MEDIA_BUS_FMT_SGBRG12_1X12, - .depth = 16, + .bpp = 12, }, { .fourcc = V4L2_PIX_FMT_SGRBG12, .code = MEDIA_BUS_FMT_SGRBG12_1X12, - .depth = 16, + .bpp = 12, }, { .fourcc = V4L2_PIX_FMT_SRGGB12, .code = MEDIA_BUS_FMT_SRGGB12_1X12, - .depth = 16, + .bpp = 12, }, }; @@ -220,20 +221,118 @@ struct cal_dmaqueue { int ini_jiffies; }; -struct cm_data { +struct cc_data { void __iomem *base; struct resource *res; - unsigned int camerrx_control; - struct platform_device *pdev; }; -struct cc_data { - void __iomem *base; - struct resource *res; +/* CTRL_CORE_CAMERRX_CONTROL register field id */ +enum cal_camerarx_field { + F_CTRLCLKEN, + F_CAMMODE, + F_LANEENABLE, + F_CSI_MODE, - struct platform_device *pdev; + F_MAX_FIELDS, +}; + +struct cal_csi2_phy { + struct regmap_field *fields[F_MAX_FIELDS]; + struct reg_field *base_fields; + const int num_lanes; +}; + +struct cal_data { + const int num_csi2_phy; + struct cal_csi2_phy *csi2_phy_core; + + const unsigned int flags; +}; + +static struct reg_field dra72x_ctrl_core_csi0_reg_fields[F_MAX_FIELDS] = { + [F_CTRLCLKEN] = REG_FIELD(0, 10, 10), + [F_CAMMODE] = REG_FIELD(0, 11, 12), + [F_LANEENABLE] = REG_FIELD(0, 13, 16), + [F_CSI_MODE] = REG_FIELD(0, 17, 17), +}; + +static struct reg_field dra72x_ctrl_core_csi1_reg_fields[F_MAX_FIELDS] = { + [F_CTRLCLKEN] = REG_FIELD(0, 0, 0), + [F_CAMMODE] = REG_FIELD(0, 1, 2), + [F_LANEENABLE] = REG_FIELD(0, 3, 4), + [F_CSI_MODE] = REG_FIELD(0, 5, 5), +}; + +static struct cal_csi2_phy dra72x_cal_csi_phy[] = { + { + .base_fields = dra72x_ctrl_core_csi0_reg_fields, + .num_lanes = 4, + }, + { + .base_fields = dra72x_ctrl_core_csi1_reg_fields, + .num_lanes = 2, + }, +}; + +static const struct cal_data dra72x_cal_data = { + .csi2_phy_core = dra72x_cal_csi_phy, + .num_csi2_phy = ARRAY_SIZE(dra72x_cal_csi_phy), +}; + +static const struct cal_data dra72x_es1_cal_data = { + .csi2_phy_core = dra72x_cal_csi_phy, + .num_csi2_phy = ARRAY_SIZE(dra72x_cal_csi_phy), + .flags = DRA72_CAL_PRE_ES2_LDO_DISABLE, +}; + +static struct reg_field dra76x_ctrl_core_csi0_reg_fields[F_MAX_FIELDS] = { + [F_CTRLCLKEN] = REG_FIELD(0, 8, 8), + [F_CAMMODE] = REG_FIELD(0, 9, 10), + [F_CSI_MODE] = REG_FIELD(0, 11, 11), + [F_LANEENABLE] = REG_FIELD(0, 27, 31), +}; + +static struct reg_field dra76x_ctrl_core_csi1_reg_fields[F_MAX_FIELDS] = { + [F_CTRLCLKEN] = REG_FIELD(0, 0, 0), + [F_CAMMODE] = REG_FIELD(0, 1, 2), + [F_CSI_MODE] = REG_FIELD(0, 3, 3), + [F_LANEENABLE] = REG_FIELD(0, 24, 26), +}; + +static struct cal_csi2_phy dra76x_cal_csi_phy[] = { + { + .base_fields = dra76x_ctrl_core_csi0_reg_fields, + .num_lanes = 5, + }, + { + .base_fields = dra76x_ctrl_core_csi1_reg_fields, + .num_lanes = 3, + }, +}; + +static const struct cal_data dra76x_cal_data = { + .csi2_phy_core = dra76x_cal_csi_phy, + .num_csi2_phy = ARRAY_SIZE(dra76x_cal_csi_phy), +}; + +static struct reg_field am654_ctrl_core_csi0_reg_fields[F_MAX_FIELDS] = { + [F_CTRLCLKEN] = REG_FIELD(0, 15, 15), + [F_CAMMODE] = REG_FIELD(0, 24, 25), + [F_LANEENABLE] = REG_FIELD(0, 0, 4), +}; + +static struct cal_csi2_phy am654_cal_csi_phy[] = { + { + .base_fields = am654_ctrl_core_csi0_reg_fields, + .num_lanes = 5, + }, +}; + +static const struct cal_data am654_cal_data = { + .csi2_phy_core = am654_cal_csi_phy, + .num_csi2_phy = ARRAY_SIZE(am654_cal_csi_phy), }; /* @@ -247,8 +346,15 @@ struct cal_dev { struct platform_device *pdev; struct v4l2_device v4l2_dev; + /* Controller flags for special cases */ + unsigned int flags; + + const struct cal_data *data; + /* Control Module handle */ - struct cm_data *cm; + struct regmap *syscon_camerrx; + u32 syscon_camerrx_offset; + /* Camera Core Module handle */ struct cc_data *cc[CAL_NUM_CSI2_PORTS]; @@ -359,73 +465,115 @@ static inline void set_field(u32 *valp, u32 field, u32 mask) *valp = val; } -/* - * Control Module block access - */ -static struct cm_data *cm_create(struct cal_dev *dev) +static u32 cal_data_get_phy_max_lanes(struct cal_ctx *ctx) { - struct platform_device *pdev = dev->pdev; - struct cm_data *cm; + struct cal_dev *dev = ctx->dev; + u32 phy_id = ctx->csi2_port - 1; - cm = devm_kzalloc(&pdev->dev, sizeof(*cm), GFP_KERNEL); - if (!cm) - return ERR_PTR(-ENOMEM); + return dev->data->csi2_phy_core[phy_id].num_lanes; +} + +static u32 cal_data_get_num_csi2_phy(struct cal_dev *dev) +{ + return dev->data->num_csi2_phy; +} + +static int cal_camerarx_regmap_init(struct cal_dev *dev) +{ + struct reg_field *field; + struct cal_csi2_phy *phy; + int i, j; + + if (!dev->data) + return -EINVAL; + + for (i = 0; i < cal_data_get_num_csi2_phy(dev); i++) { + phy = &dev->data->csi2_phy_core[i]; + for (j = 0; j < F_MAX_FIELDS; j++) { + field = &phy->base_fields[j]; + /* + * Here we update the reg offset with the + * value found in DT + */ + field->reg = dev->syscon_camerrx_offset; + phy->fields[j] = + devm_regmap_field_alloc(&dev->pdev->dev, + dev->syscon_camerrx, + *field); + if (IS_ERR(phy->fields[j])) { + cal_err(dev, "Unable to allocate regmap fields\n"); + return PTR_ERR(phy->fields[j]); + } + } + } + return 0; +} + +static const struct regmap_config cal_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; - cm->res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "camerrx_control"); - cm->base = devm_ioremap_resource(&pdev->dev, cm->res); - if (IS_ERR(cm->base)) { +static struct regmap *cal_get_camerarx_regmap(struct cal_dev *dev) +{ + struct platform_device *pdev = dev->pdev; + struct regmap *regmap; + void __iomem *base; + u32 reg_io_width; + struct regmap_config r_config = cal_regmap_config; + struct resource *res; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "camerrx_control"); + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) { cal_err(dev, "failed to ioremap\n"); - return ERR_CAST(cm->base); + return ERR_CAST(base); } cal_dbg(1, dev, "ioresource %s at %pa - %pa\n", - cm->res->name, &cm->res->start, &cm->res->end); + res->name, &res->start, &res->end); - return cm; + reg_io_width = 4; + r_config.reg_stride = reg_io_width; + r_config.val_bits = reg_io_width * 8; + r_config.max_register = resource_size(res) - reg_io_width; + + regmap = regmap_init_mmio(NULL, base, &r_config); + if (IS_ERR(regmap)) + pr_err("regmap init failed\n"); + + return regmap; } +/* + * Control Module CAMERARX block access + */ static void camerarx_phy_enable(struct cal_ctx *ctx) { - u32 val; - - if (!ctx->dev->cm->base) { - ctx_err(ctx, "cm not mapped\n"); - return; - } - - val = reg_read(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL); - if (ctx->csi2_port == 1) { - set_field(&val, 1, CM_CAMERRX_CTRL_CSI0_CTRLCLKEN_MASK); - set_field(&val, 0, CM_CAMERRX_CTRL_CSI0_CAMMODE_MASK); - /* enable all lanes by default */ - set_field(&val, 0xf, CM_CAMERRX_CTRL_CSI0_LANEENABLE_MASK); - set_field(&val, 1, CM_CAMERRX_CTRL_CSI0_MODE_MASK); - } else if (ctx->csi2_port == 2) { - set_field(&val, 1, CM_CAMERRX_CTRL_CSI1_CTRLCLKEN_MASK); - set_field(&val, 0, CM_CAMERRX_CTRL_CSI1_CAMMODE_MASK); - /* enable all lanes by default */ - set_field(&val, 0x3, CM_CAMERRX_CTRL_CSI1_LANEENABLE_MASK); - set_field(&val, 1, CM_CAMERRX_CTRL_CSI1_MODE_MASK); - } - reg_write(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL, val); + struct cal_csi2_phy *phy; + u32 phy_id = ctx->csi2_port - 1; + u32 max_lanes; + + phy = &ctx->dev->data->csi2_phy_core[phy_id]; + regmap_field_write(phy->fields[F_CAMMODE], 0); + /* Always enable all lanes at the phy control level */ + max_lanes = (1 << cal_data_get_phy_max_lanes(ctx)) - 1; + regmap_field_write(phy->fields[F_LANEENABLE], max_lanes); + /* F_CSI_MODE is not present on every architecture */ + if (phy->fields[F_CSI_MODE]) + regmap_field_write(phy->fields[F_CSI_MODE], 1); + regmap_field_write(phy->fields[F_CTRLCLKEN], 1); } static void camerarx_phy_disable(struct cal_ctx *ctx) { - u32 val; - - if (!ctx->dev->cm->base) { - ctx_err(ctx, "cm not mapped\n"); - return; - } + struct cal_csi2_phy *phy; + u32 phy_id = ctx->csi2_port - 1; - val = reg_read(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL); - if (ctx->csi2_port == 1) - set_field(&val, 0x0, CM_CAMERRX_CTRL_CSI0_CTRLCLKEN_MASK); - else if (ctx->csi2_port == 2) - set_field(&val, 0x0, CM_CAMERRX_CTRL_CSI1_CTRLCLKEN_MASK); - reg_write(ctx->dev->cm, CM_CTRL_CORE_CAMERRX_CONTROL, val); + phy = &ctx->dev->data->csi2_phy_core[phy_id]; + regmap_field_write(phy->fields[F_CTRLCLKEN], 0); } /* @@ -474,9 +622,52 @@ static void cal_get_hwinfo(struct cal_dev *dev) hwinfo); } -static inline int cal_runtime_get(struct cal_dev *dev) +/* + * Errata i913: CSI2 LDO Needs to be disabled when module is powered on + * + * Enabling CSI2 LDO shorts it to core supply. It is crucial the 2 CSI2 + * LDOs on the device are disabled if CSI-2 module is powered on + * (0x4845 B304 | 0x4845 B384 [28:27] = 0x1) or in ULPS (0x4845 B304 + * | 0x4845 B384 [28:27] = 0x2) mode. Common concerns include: high + * current draw on the module supply in active mode. + * + * Errata does not apply when CSI-2 module is powered off + * (0x4845 B304 | 0x4845 B384 [28:27] = 0x0). + * + * SW Workaround: + * Set the following register bits to disable the LDO, + * which is essentially CSI2 REG10 bit 6: + * + * Core 0: 0x4845 B828 = 0x0000 0040 + * Core 1: 0x4845 B928 = 0x0000 0040 + */ +static void i913_errata(struct cal_dev *dev, unsigned int port) +{ + u32 reg10 = reg_read(dev->cc[port], CAL_CSI2_PHY_REG10); + + set_field(®10, CAL_CSI2_PHY_REG0_HSCLOCKCONFIG_DISABLE, + CAL_CSI2_PHY_REG10_I933_LDO_DISABLE_MASK); + + cal_dbg(1, dev, "CSI2_%d_REG10 = 0x%08x\n", port, reg10); + reg_write(dev->cc[port], CAL_CSI2_PHY_REG10, reg10); +} + +static int cal_runtime_get(struct cal_dev *dev) { - return pm_runtime_get_sync(&dev->pdev->dev); + int r; + + r = pm_runtime_get_sync(&dev->pdev->dev); + + if (dev->flags & DRA72_CAL_PRE_ES2_LDO_DISABLE) { + /* + * Apply errata on both port eveytime we (re-)enable + * the clock + */ + i913_errata(dev, 0); + i913_errata(dev, 1); + } + + return r; } static inline void cal_runtime_put(struct cal_dev *dev) @@ -508,12 +699,6 @@ static void cal_quickdump_regs(struct cal_dev *dev) resource_size(dev->ctx[1]->cc->res), false); } - - cal_info(dev, "CAMERRX_Control Registers @ %pa:\n", - &dev->cm->res->start); - print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 4, - (__force const void *)dev->cm->base, - resource_size(dev->cm->res), false); } /* @@ -551,29 +736,76 @@ static void disable_irqs(struct cal_ctx *ctx) reg_write(ctx->dev, CAL_CSI2_VC_IRQENABLE(1), 0); } -static void csi2_init(struct cal_ctx *ctx) +static void csi2_phy_config(struct cal_ctx *ctx); + +static void csi2_phy_init(struct cal_ctx *ctx) { int i; u32 val; + /* Steps + * 1. Configure D-PHY mode and enable required lanes + * 2. Reset complex IO - Wait for completion of reset + * Note if the external sensor is not sending byte clock, + * the reset will timeout + * 3 Program Stop States + * A. Program THS_TERM, THS_SETTLE, etc... Timings parameters + * in terms of DDR clock periods + * B. Enable stop state transition timeouts + * 4.Force FORCERXMODE + * D. Enable pull down using pad control + * E. Power up PHY + * F. Wait for power up completion + * G. Wait for all enabled lane to reach stop state + * H. Disable pull down using pad control + */ + + /* 1. Configure D-PHY mode and enable required lanes */ + camerarx_phy_enable(ctx); + + /* 2. Reset complex IO - Do not wait for reset completion */ + val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)); + set_field(&val, CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_OPERATIONAL, + CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK); + reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val); + ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x De-assert Complex IO Reset\n", + ctx->csi2_port, + reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port))); + + /* Dummy read to allow SCP to complete */ + val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)); + + /* 3.A. Program Phy Timing Parameters */ + csi2_phy_config(ctx); + + /* 3.B. Program Stop States */ val = reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port)); set_field(&val, CAL_GEN_ENABLE, - CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK); - set_field(&val, CAL_GEN_ENABLE, CAL_CSI2_TIMING_STOP_STATE_X16_IO1_MASK); set_field(&val, CAL_GEN_DISABLE, CAL_CSI2_TIMING_STOP_STATE_X4_IO1_MASK); set_field(&val, 407, CAL_CSI2_TIMING_STOP_STATE_COUNTER_IO1_MASK); reg_write(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port), val); - ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x\n", ctx->csi2_port, + ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x Stop States\n", + ctx->csi2_port, + reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port))); + + /* 4. Force FORCERXMODE */ + val = reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port)); + set_field(&val, CAL_GEN_ENABLE, + CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK); + reg_write(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port), val); + ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x Force RXMODE\n", + ctx->csi2_port, reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port))); + /* E. Power up the PHY using the complex IO */ val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)); - set_field(&val, CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_OPERATIONAL, - CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK); set_field(&val, CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_STATE_ON, CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_MASK); reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val); + + /* F. Wait for power up completion */ for (i = 0; i < 10; i++) { if (reg_read_field(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), @@ -582,18 +814,104 @@ static void csi2_init(struct cal_ctx *ctx) break; usleep_range(1000, 1100); } - ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x\n", ctx->csi2_port, - reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port))); + ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Powered UP %s\n", + ctx->csi2_port, + reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)), + (i >= 10) ? "(timeout)" : ""); +} - val = reg_read(ctx->dev, CAL_CTRL); - set_field(&val, CAL_CTRL_BURSTSIZE_BURST128, CAL_CTRL_BURSTSIZE_MASK); - set_field(&val, 0xF, CAL_CTRL_TAGCNT_MASK); - set_field(&val, CAL_CTRL_POSTED_WRITES_NONPOSTED, - CAL_CTRL_POSTED_WRITES_MASK); - set_field(&val, 0xFF, CAL_CTRL_MFLAGL_MASK); - set_field(&val, 0xFF, CAL_CTRL_MFLAGH_MASK); - reg_write(ctx->dev, CAL_CTRL, val); - ctx_dbg(3, ctx, "CAL_CTRL = 0x%08x\n", reg_read(ctx->dev, CAL_CTRL)); +static void csi2_wait_for_phy(struct cal_ctx *ctx) +{ + int i; + + /* Steps + * 2. Wait for completion of reset + * Note if the external sensor is not sending byte clock, + * the reset will timeout + * 4.Force FORCERXMODE + * G. Wait for all enabled lane to reach stop state + * H. Disable pull down using pad control + */ + + /* 2. Wait for reset completion */ + for (i = 0; i < 250; i++) { + if (reg_read_field(ctx->dev, + CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), + CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_MASK) == + CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_RESETCOMPLETED) + break; + usleep_range(1000, 1100); + } + ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Complex IO Reset Done (%d) %s\n", + ctx->csi2_port, + reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)), i, + (i >= 250) ? "(timeout)" : ""); + + /* 4. G. Wait for all enabled lane to reach stop state */ + for (i = 0; i < 10; i++) { + if (reg_read_field(ctx->dev, + CAL_CSI2_TIMING(ctx->csi2_port), + CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK) == + CAL_GEN_DISABLE) + break; + usleep_range(1000, 1100); + } + ctx_dbg(3, ctx, "CAL_CSI2_TIMING(%d) = 0x%08x Stop State Reached %s\n", + ctx->csi2_port, + reg_read(ctx->dev, CAL_CSI2_TIMING(ctx->csi2_port)), + (i >= 10) ? "(timeout)" : ""); + + ctx_dbg(1, ctx, "CSI2_%d_REG1 = 0x%08x (Bit(31,28) should be set!)\n", + (ctx->csi2_port - 1), reg_read(ctx->cc, CAL_CSI2_PHY_REG1)); +} + +static void csi2_phy_deinit(struct cal_ctx *ctx) +{ + int i; + u32 val; + + /* Power down the PHY using the complex IO */ + val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)); + set_field(&val, CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_STATE_OFF, + CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_MASK); + reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val); + + /* Wait for power down completion */ + for (i = 0; i < 10; i++) { + if (reg_read_field(ctx->dev, + CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), + CAL_CSI2_COMPLEXIO_CFG_PWR_STATUS_MASK) == + CAL_CSI2_COMPLEXIO_CFG_PWR_STATUS_STATE_OFF) + break; + usleep_range(1000, 1100); + } + ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Powered Down %s\n", + ctx->csi2_port, + reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)), + (i >= 10) ? "(timeout)" : ""); + + /* Assert Comple IO Reset */ + val = reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)); + set_field(&val, CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL, + CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK); + reg_write(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), val); + + /* Wait for power down completion */ + for (i = 0; i < 10; i++) { + if (reg_read_field(ctx->dev, + CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port), + CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_MASK) == + CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_RESETONGOING) + break; + usleep_range(1000, 1100); + } + ctx_dbg(3, ctx, "CAL_CSI2_COMPLEXIO_CFG(%d) = 0x%08x Complex IO in Reset (%d) %s\n", + ctx->csi2_port, + reg_read(ctx->dev, CAL_CSI2_COMPLEXIO_CFG(ctx->csi2_port)), i, + (i >= 10) ? "(timeout)" : ""); + + /* Disable the phy */ + camerarx_phy_disable(ctx); } static void csi2_lane_config(struct cal_ctx *ctx) @@ -665,13 +983,48 @@ static void csi2_ctx_config(struct cal_ctx *ctx) static void pix_proc_config(struct cal_ctx *ctx) { - u32 val; + u32 val, extract, pack; + + switch (ctx->fmt->bpp) { + case 8: + extract = CAL_PIX_PROC_EXTRACT_B8; + pack = CAL_PIX_PROC_PACK_B8; + break; + case 10: + extract = CAL_PIX_PROC_EXTRACT_B10_MIPI; + pack = CAL_PIX_PROC_PACK_B16; + break; + case 12: + extract = CAL_PIX_PROC_EXTRACT_B12_MIPI; + pack = CAL_PIX_PROC_PACK_B16; + break; + case 16: + extract = CAL_PIX_PROC_EXTRACT_B16_LE; + pack = CAL_PIX_PROC_PACK_B16; + break; + default: + /* + * If you see this warning then it means that you added + * some new entry in the cal_formats[] array with a different + * bit per pixel values then the one supported below. + * Either add support for the new bpp value below or adjust + * the new entry to use one of the value below. + * + * Instead of failing here just use 8 bpp as a default. + */ + dev_warn_once(&ctx->dev->pdev->dev, + "%s:%d:%s: bpp:%d unsupported! Overwritten with 8.\n", + __FILE__, __LINE__, __func__, ctx->fmt->bpp); + extract = CAL_PIX_PROC_EXTRACT_B8; + pack = CAL_PIX_PROC_PACK_B8; + break; + } val = reg_read(ctx->dev, CAL_PIX_PROC(ctx->csi2_port)); - set_field(&val, CAL_PIX_PROC_EXTRACT_B8, CAL_PIX_PROC_EXTRACT_MASK); + set_field(&val, extract, CAL_PIX_PROC_EXTRACT_MASK); set_field(&val, CAL_PIX_PROC_DPCMD_BYPASS, CAL_PIX_PROC_DPCMD_MASK); set_field(&val, CAL_PIX_PROC_DPCME_BYPASS, CAL_PIX_PROC_DPCME_MASK); - set_field(&val, CAL_PIX_PROC_PACK_B8, CAL_PIX_PROC_PACK_MASK); + set_field(&val, pack, CAL_PIX_PROC_PACK_MASK); set_field(&val, ctx->csi2_port, CAL_PIX_PROC_CPORT_MASK); set_field(&val, CAL_GEN_ENABLE, CAL_PIX_PROC_EN_MASK); reg_write(ctx->dev, CAL_PIX_PROC(ctx->csi2_port), val); @@ -680,12 +1033,13 @@ static void pix_proc_config(struct cal_ctx *ctx) } static void cal_wr_dma_config(struct cal_ctx *ctx, - unsigned int width) + unsigned int width, unsigned int height) { u32 val; val = reg_read(ctx->dev, CAL_WR_DMA_CTRL(ctx->csi2_port)); set_field(&val, ctx->csi2_port, CAL_WR_DMA_CTRL_CPORT_MASK); + set_field(&val, height, CAL_WR_DMA_CTRL_YSIZE_MASK); set_field(&val, CAL_WR_DMA_CTRL_DTAG_PIX_DAT, CAL_WR_DMA_CTRL_DTAG_MASK); set_field(&val, CAL_WR_DMA_CTRL_MODE_CONST, @@ -720,6 +1074,16 @@ static void cal_wr_dma_config(struct cal_ctx *ctx, reg_write(ctx->dev, CAL_WR_DMA_XSIZE(ctx->csi2_port), val); ctx_dbg(3, ctx, "CAL_WR_DMA_XSIZE(%d) = 0x%08x\n", ctx->csi2_port, reg_read(ctx->dev, CAL_WR_DMA_XSIZE(ctx->csi2_port))); + + val = reg_read(ctx->dev, CAL_CTRL); + set_field(&val, CAL_CTRL_BURSTSIZE_BURST128, CAL_CTRL_BURSTSIZE_MASK); + set_field(&val, 0xF, CAL_CTRL_TAGCNT_MASK); + set_field(&val, CAL_CTRL_POSTED_WRITES_NONPOSTED, + CAL_CTRL_POSTED_WRITES_MASK); + set_field(&val, 0xFF, CAL_CTRL_MFLAGL_MASK); + set_field(&val, 0xFF, CAL_CTRL_MFLAGH_MASK); + reg_write(ctx->dev, CAL_CTRL, val); + ctx_dbg(3, ctx, "CAL_CTRL = 0x%08x\n", reg_read(ctx->dev, CAL_CTRL)); } static void cal_wr_dma_addr(struct cal_ctx *ctx, unsigned int dmaaddr) @@ -733,41 +1097,28 @@ static void cal_wr_dma_addr(struct cal_ctx *ctx, unsigned int dmaaddr) #define TCLK_TERM 0 #define TCLK_MISS 1 #define TCLK_SETTLE 14 -#define THS_SETTLE 15 static void csi2_phy_config(struct cal_ctx *ctx) { unsigned int reg0, reg1; unsigned int ths_term, ths_settle; - unsigned int ddrclkperiod_us; + unsigned int csi2_ddrclk_khz; + struct v4l2_fwnode_bus_mipi_csi2 *mipi_csi2 = + &ctx->endpoint.bus.mipi_csi2; + u32 num_lanes = mipi_csi2->num_data_lanes; - /* - * THS_TERM: Programmed value = floor(20 ns/DDRClk period) - 2. - */ - ddrclkperiod_us = ctx->external_rate / 2000000; - ddrclkperiod_us = 1000000 / ddrclkperiod_us; - ctx_dbg(1, ctx, "ddrclkperiod_us: %d\n", ddrclkperiod_us); + /* DPHY timing configuration */ + /* CSI-2 is DDR and we only count used lanes. */ + csi2_ddrclk_khz = ctx->external_rate / 1000 + / (2 * num_lanes) * ctx->fmt->bpp; + ctx_dbg(1, ctx, "csi2_ddrclk_khz: %d\n", csi2_ddrclk_khz); - ths_term = 20000 / ddrclkperiod_us; - ths_term = (ths_term >= 2) ? ths_term - 2 : ths_term; + /* THS_TERM: Programmed value = floor(20 ns/DDRClk period) */ + ths_term = 20 * csi2_ddrclk_khz / 1000000; ctx_dbg(1, ctx, "ths_term: %d (0x%02x)\n", ths_term, ths_term); - /* - * THS_SETTLE: Programmed value = floor(176.3 ns/CtrlClk period) - 1. - * Since CtrlClk is fixed at 96Mhz then we get - * ths_settle = floor(176.3 / 10.416) - 1 = 15 - * If we ever switch to a dynamic clock then this code might be useful - * - * unsigned int ctrlclkperiod_us; - * ctrlclkperiod_us = 96000000 / 1000000; - * ctrlclkperiod_us = 1000000 / ctrlclkperiod_us; - * ctx_dbg(1, ctx, "ctrlclkperiod_us: %d\n", ctrlclkperiod_us); - - * ths_settle = 176300 / ctrlclkperiod_us; - * ths_settle = (ths_settle > 1) ? ths_settle - 1 : ths_settle; - */ - - ths_settle = THS_SETTLE; + /* THS_SETTLE: Programmed value = floor(105 ns/DDRClk period) + 4 */ + ths_settle = (105 * csi2_ddrclk_khz / 1000000) + 4; ctx_dbg(1, ctx, "ths_settle: %d (0x%02x)\n", ths_settle, ths_settle); reg0 = reg_read(ctx->cc, CAL_CSI2_PHY_REG0); @@ -979,15 +1330,25 @@ static int cal_calc_format_size(struct cal_ctx *ctx, const struct cal_fmt *fmt, struct v4l2_format *f) { + u32 bpl, max_width; + if (!fmt) { ctx_dbg(3, ctx, "No cal_fmt provided!\n"); return -EINVAL; } - v4l_bound_align_image(&f->fmt.pix.width, 48, MAX_WIDTH, 2, - &f->fmt.pix.height, 32, MAX_HEIGHT, 0, 0); - f->fmt.pix.bytesperline = bytes_per_line(f->fmt.pix.width, - fmt->depth >> 3); + /* + * Maximum width is bound by the DMA max width in bytes. + * We need to recalculate the actual maxi width depending on the + * number of bytes per pixels required. + */ + max_width = MAX_WIDTH_BYTES / (ALIGN(fmt->bpp, 8) >> 3); + v4l_bound_align_image(&f->fmt.pix.width, 48, max_width, 2, + &f->fmt.pix.height, 32, MAX_HEIGHT_LINES, 0, 0); + + bpl = (f->fmt.pix.width * ALIGN(fmt->bpp, 8)) >> 3; + f->fmt.pix.bytesperline = ALIGN(bpl, 16); + f->fmt.pix.sizeimage = f->fmt.pix.height * f->fmt.pix.bytesperline; @@ -1132,6 +1493,7 @@ static int cal_enum_framesizes(struct file *file, void *fh, fse.index = fsize->index; fse.pad = 0; fse.code = fmt->code; + fse.which = V4L2_SUBDEV_FORMAT_ACTIVE; ret = v4l2_subdev_call(ctx->sensor, pad, enum_frame_size, NULL, &fse); if (ret) @@ -1299,36 +1661,50 @@ static int cal_start_streaming(struct vb2_queue *vq, unsigned int count) if (ret < 0) goto err; + ret = v4l2_subdev_call(ctx->sensor, core, s_power, 1); + if (ret < 0 && ret != -ENOIOCTLCMD && ret != -ENODEV) { + ctx_err(ctx, "power on failed in subdev\n"); + goto err; + } + cal_runtime_get(ctx->dev); - enable_irqs(ctx); - camerarx_phy_enable(ctx); - csi2_init(ctx); - csi2_phy_config(ctx); - csi2_lane_config(ctx); csi2_ctx_config(ctx); pix_proc_config(ctx); - cal_wr_dma_config(ctx, ctx->v_fmt.fmt.pix.bytesperline); - cal_wr_dma_addr(ctx, addr); - csi2_ppi_enable(ctx); + cal_wr_dma_config(ctx, ctx->v_fmt.fmt.pix.bytesperline, + ctx->v_fmt.fmt.pix.height); + csi2_lane_config(ctx); + + enable_irqs(ctx); + csi2_phy_init(ctx); ret = v4l2_subdev_call(ctx->sensor, video, s_stream, 1); if (ret) { + v4l2_subdev_call(ctx->sensor, core, s_power, 0); ctx_err(ctx, "stream on failed in subdev\n"); cal_runtime_put(ctx->dev); goto err; } + csi2_wait_for_phy(ctx); + cal_wr_dma_addr(ctx, addr); + csi2_ppi_enable(ctx); + if (debug >= 4) cal_quickdump_regs(ctx->dev); return 0; err: + spin_lock_irqsave(&ctx->slock, flags); + vb2_buffer_done(&ctx->cur_frm->vb.vb2_buf, VB2_BUF_STATE_QUEUED); + ctx->cur_frm = NULL; + ctx->next_frm = NULL; list_for_each_entry_safe(buf, tmp, &dma_q->active, list) { list_del(&buf->list); vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED); } + spin_unlock_irqrestore(&ctx->slock, flags); return ret; } @@ -1338,12 +1714,18 @@ static void cal_stop_streaming(struct vb2_queue *vq) struct cal_dmaqueue *dma_q = &ctx->vidq; struct cal_buffer *buf, *tmp; unsigned long flags; + int ret; + + csi2_ppi_disable(ctx); + disable_irqs(ctx); + csi2_phy_deinit(ctx); if (v4l2_subdev_call(ctx->sensor, video, s_stream, 0)) ctx_err(ctx, "stream off failed in subdev\n"); - csi2_ppi_disable(ctx); - disable_irqs(ctx); + ret = v4l2_subdev_call(ctx->sensor, core, s_power, 0); + if (ret < 0 && ret != -ENOIOCTLCMD && ret != -ENODEV) + ctx_err(ctx, "power off failed in subdev\n"); /* Release all active buffers */ spin_lock_irqsave(&ctx->slock, flags); @@ -1399,6 +1781,7 @@ static const struct v4l2_ioctl_ops cal_ioctl_ops = { .vidioc_querybuf = vb2_ioctl_querybuf, .vidioc_qbuf = vb2_ioctl_qbuf, .vidioc_dqbuf = vb2_ioctl_dqbuf, + .vidioc_expbuf = vb2_ioctl_expbuf, .vidioc_enum_input = cal_enum_input, .vidioc_g_input = cal_g_input, .vidioc_s_input = cal_s_input, @@ -1451,6 +1834,7 @@ static int cal_async_bound(struct v4l2_async_notifier *notifier, memset(&mbus_code, 0, sizeof(mbus_code)); mbus_code.index = j; + mbus_code.which = V4L2_SUBDEV_FORMAT_ACTIVE; ret = v4l2_subdev_call(subdev, pad, enum_mbus_code, NULL, &mbus_code); if (ret) @@ -1804,10 +2188,15 @@ err_exit: return NULL; } +static const struct of_device_id cal_of_match[]; + static int cal_probe(struct platform_device *pdev) { struct cal_dev *dev; struct cal_ctx *ctx; + struct device_node *parent = pdev->dev.of_node; + struct regmap *syscon_camerrx = NULL; + u32 syscon_camerrx_offset = 0; int ret; int irq; int i; @@ -1816,6 +2205,14 @@ static int cal_probe(struct platform_device *pdev) if (!dev) return -ENOMEM; + dev->data = of_device_get_match_data(&pdev->dev); + if (!dev->data) { + dev_err(&pdev->dev, "Could not get feature data based on compatible version\n"); + return -ENODEV; + } + + dev->flags = dev->data->flags; + /* set pseudo v4l2 device name so we can use v4l2_printk */ strscpy(dev->v4l2_dev.name, CAL_MODULE_NAME, sizeof(dev->v4l2_dev.name)); @@ -1823,6 +2220,38 @@ static int cal_probe(struct platform_device *pdev) /* save pdev pointer */ dev->pdev = pdev; + syscon_camerrx = syscon_regmap_lookup_by_phandle(parent, + "ti,camerrx-control"); + ret = of_property_read_u32_index(parent, "ti,camerrx-control", 1, + &syscon_camerrx_offset); + if (IS_ERR(syscon_camerrx)) + ret = PTR_ERR(syscon_camerrx); + if (ret) { + dev_warn(&pdev->dev, "failed to get ti,camerrx-control: %d\n", + ret); + + /* + * Backward DTS compatibility. + * If syscon entry is not present then check if the + * camerrx_control resource is present. + */ + syscon_camerrx = cal_get_camerarx_regmap(dev); + if (IS_ERR(syscon_camerrx)) { + dev_err(&pdev->dev, "failed to get camerrx_control regmap\n"); + return PTR_ERR(syscon_camerrx); + } + /* In this case the base already point to the direct + * CM register so no need for an offset + */ + syscon_camerrx_offset = 0; + } + + dev->syscon_camerrx = syscon_camerrx; + dev->syscon_camerrx_offset = syscon_camerrx_offset; + ret = cal_camerarx_regmap_init(dev); + if (ret) + return ret; + dev->res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cal_top"); dev->base = devm_ioremap_resource(&pdev->dev, dev->res); @@ -1841,23 +2270,24 @@ static int cal_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dev); - dev->cm = cm_create(dev); - if (IS_ERR(dev->cm)) - return PTR_ERR(dev->cm); - dev->cc[0] = cc_create(dev, 0); if (IS_ERR(dev->cc[0])) return PTR_ERR(dev->cc[0]); - dev->cc[1] = cc_create(dev, 1); - if (IS_ERR(dev->cc[1])) - return PTR_ERR(dev->cc[1]); + if (cal_data_get_num_csi2_phy(dev) > 1) { + dev->cc[1] = cc_create(dev, 1); + if (IS_ERR(dev->cc[1])) + return PTR_ERR(dev->cc[1]); + } else { + dev->cc[1] = NULL; + } dev->ctx[0] = NULL; dev->ctx[1] = NULL; dev->ctx[0] = cal_create_instance(dev, 0); - dev->ctx[1] = cal_create_instance(dev, 1); + if (cal_data_get_num_csi2_phy(dev) > 1) + dev->ctx[1] = cal_create_instance(dev, 1); if (!dev->ctx[0] && !dev->ctx[1]) { cal_err(dev, "Neither port is configured, no point in staying up\n"); return -ENODEV; @@ -1924,7 +2354,22 @@ static int cal_remove(struct platform_device *pdev) #if defined(CONFIG_OF) static const struct of_device_id cal_of_match[] = { - { .compatible = "ti,dra72-cal", }, + { + .compatible = "ti,dra72-cal", + .data = (void *)&dra72x_cal_data, + }, + { + .compatible = "ti,dra72-pre-es2-cal", + .data = (void *)&dra72x_es1_cal_data, + }, + { + .compatible = "ti,dra76-cal", + .data = (void *)&dra76x_cal_data, + }, + { + .compatible = "ti,am654-cal", + .data = (void *)&am654_cal_data, + }, {}, }; MODULE_DEVICE_TABLE(of, cal_of_match); diff --git a/drivers/media/platform/ti-vpe/cal_regs.h b/drivers/media/platform/ti-vpe/cal_regs.h index 68cfc922b422..0b76d1186074 100644 --- a/drivers/media/platform/ti-vpe/cal_regs.h +++ b/drivers/media/platform/ti-vpe/cal_regs.h @@ -10,6 +10,30 @@ #ifndef __TI_CAL_REGS_H #define __TI_CAL_REGS_H +/* + * struct cal_dev.flags possibilities + * + * DRA72_CAL_PRE_ES2_LDO_DISABLE: + * Errata i913: CSI2 LDO Needs to be disabled when module is powered on + * + * Enabling CSI2 LDO shorts it to core supply. It is crucial the 2 CSI2 + * LDOs on the device are disabled if CSI-2 module is powered on + * (0x4845 B304 | 0x4845 B384 [28:27] = 0x1) or in ULPS (0x4845 B304 + * | 0x4845 B384 [28:27] = 0x2) mode. Common concerns include: high + * current draw on the module supply in active mode. + * + * Errata does not apply when CSI-2 module is powered off + * (0x4845 B304 | 0x4845 B384 [28:27] = 0x0). + * + * SW Workaround: + * Set the following register bits to disable the LDO, + * which is essentially CSI2 REG10 bit 6: + * + * Core 0: 0x4845 B828 = 0x0000 0040 + * Core 1: 0x4845 B928 = 0x0000 0040 + */ +#define DRA72_CAL_PRE_ES2_LDO_DISABLE BIT(0) + #define CAL_NUM_CSI2_PORTS 2 /* CAL register offsets */ @@ -71,6 +95,7 @@ #define CAL_CSI2_PHY_REG0 0x000 #define CAL_CSI2_PHY_REG1 0x004 #define CAL_CSI2_PHY_REG2 0x008 +#define CAL_CSI2_PHY_REG10 0x028 /* CAL Control Module Core Camerrx Control register offsets */ #define CM_CTRL_CORE_CAMERRX_CONTROL 0x000 @@ -110,7 +135,7 @@ #define CAL_HL_HWINFO_NPPI_CONTEXTS_EIGHT 2 #define CAL_HL_HWINFO_NPPI_CONTEXTS_RESERVED 3 -#define CAL_HL_SYSCONFIG_SOFTRESET_MASK BIT_MASK(0) +#define CAL_HL_SYSCONFIG_SOFTRESET_MASK BIT(0) #define CAL_HL_SYSCONFIG_SOFTRESET_DONE 0x0 #define CAL_HL_SYSCONFIG_SOFTRESET_PENDING 0x1 #define CAL_HL_SYSCONFIG_SOFTRESET_NOACTION 0x0 @@ -121,11 +146,11 @@ #define CAL_HL_SYSCONFIG_IDLEMODE_SMART1 2 #define CAL_HL_SYSCONFIG_IDLEMODE_SMART2 3 -#define CAL_HL_IRQ_EOI_LINE_NUMBER_MASK BIT_MASK(0) +#define CAL_HL_IRQ_EOI_LINE_NUMBER_MASK BIT(0) #define CAL_HL_IRQ_EOI_LINE_NUMBER_READ0 0 #define CAL_HL_IRQ_EOI_LINE_NUMBER_EOI0 0 -#define CAL_HL_IRQ_MASK(m) BIT_MASK(m-1) +#define CAL_HL_IRQ_MASK(m) BIT((m) - 1) #define CAL_HL_IRQ_NOACTION 0x0 #define CAL_HL_IRQ_ENABLE 0x1 #define CAL_HL_IRQ_CLEAR 0x1 @@ -133,7 +158,7 @@ #define CAL_HL_IRQ_ENABLED 0x1 #define CAL_HL_IRQ_PENDING 0x1 -#define CAL_PIX_PROC_EN_MASK BIT_MASK(0) +#define CAL_PIX_PROC_EN_MASK BIT(0) #define CAL_PIX_PROC_EXTRACT_MASK GENMASK(4, 1) #define CAL_PIX_PROC_EXTRACT_B6 0x0 #define CAL_PIX_PROC_EXTRACT_B7 0x1 @@ -179,7 +204,7 @@ #define CAL_PIX_PROC_PACK_ARGB 0x6 #define CAL_PIX_PROC_CPORT_MASK GENMASK(23, 19) -#define CAL_CTRL_POSTED_WRITES_MASK BIT_MASK(0) +#define CAL_CTRL_POSTED_WRITES_MASK BIT(0) #define CAL_CTRL_POSTED_WRITES_NONPOSTED 0 #define CAL_CTRL_POSTED_WRITES 1 #define CAL_CTRL_TAGCNT_MASK GENMASK(4, 1) @@ -190,10 +215,10 @@ #define CAL_CTRL_BURSTSIZE_BURST128 0x3 #define CAL_CTRL_LL_FORCE_STATE_MASK GENMASK(12, 7) #define CAL_CTRL_MFLAGL_MASK GENMASK(20, 13) -#define CAL_CTRL_PWRSCPCLK_MASK BIT_MASK(21) +#define CAL_CTRL_PWRSCPCLK_MASK BIT(21) #define CAL_CTRL_PWRSCPCLK_AUTO 0 #define CAL_CTRL_PWRSCPCLK_FORCE 1 -#define CAL_CTRL_RD_DMA_STALL_MASK BIT_MASK(22) +#define CAL_CTRL_RD_DMA_STALL_MASK BIT(22) #define CAL_CTRL_MFLAGH_MASK GENMASK(31, 24) #define CAL_CTRL1_PPI_GROUPING_MASK GENMASK(1, 0) @@ -218,18 +243,18 @@ #define CAL_VPORT_CTRL1_PCLK_MASK GENMASK(16, 0) #define CAL_VPORT_CTRL1_XBLK_MASK GENMASK(24, 17) #define CAL_VPORT_CTRL1_YBLK_MASK GENMASK(30, 25) -#define CAL_VPORT_CTRL1_WIDTH_MASK BIT_MASK(31) +#define CAL_VPORT_CTRL1_WIDTH_MASK BIT(31) #define CAL_VPORT_CTRL1_WIDTH_ONE 0 #define CAL_VPORT_CTRL1_WIDTH_TWO 1 #define CAL_VPORT_CTRL2_CPORT_MASK GENMASK(4, 0) -#define CAL_VPORT_CTRL2_FREERUNNING_MASK BIT_MASK(15) +#define CAL_VPORT_CTRL2_FREERUNNING_MASK BIT(15) #define CAL_VPORT_CTRL2_FREERUNNING_GATED 0 #define CAL_VPORT_CTRL2_FREERUNNING_FREE 1 -#define CAL_VPORT_CTRL2_FS_RESETS_MASK BIT_MASK(16) +#define CAL_VPORT_CTRL2_FS_RESETS_MASK BIT(16) #define CAL_VPORT_CTRL2_FS_RESETS_NO 0 #define CAL_VPORT_CTRL2_FS_RESETS_YES 1 -#define CAL_VPORT_CTRL2_FSM_RESET_MASK BIT_MASK(17) +#define CAL_VPORT_CTRL2_FSM_RESET_MASK BIT(17) #define CAL_VPORT_CTRL2_FSM_RESET_NOEFFECT 0 #define CAL_VPORT_CTRL2_FSM_RESET 1 #define CAL_VPORT_CTRL2_RDY_THR_MASK GENMASK(31, 18) @@ -237,23 +262,23 @@ #define CAL_BYS_CTRL1_PCLK_MASK GENMASK(16, 0) #define CAL_BYS_CTRL1_XBLK_MASK GENMASK(24, 17) #define CAL_BYS_CTRL1_YBLK_MASK GENMASK(30, 25) -#define CAL_BYS_CTRL1_BYSINEN_MASK BIT_MASK(31) +#define CAL_BYS_CTRL1_BYSINEN_MASK BIT(31) #define CAL_BYS_CTRL2_CPORTIN_MASK GENMASK(4, 0) #define CAL_BYS_CTRL2_CPORTOUT_MASK GENMASK(9, 5) -#define CAL_BYS_CTRL2_DUPLICATEDDATA_MASK BIT_MASK(10) +#define CAL_BYS_CTRL2_DUPLICATEDDATA_MASK BIT(10) #define CAL_BYS_CTRL2_DUPLICATEDDATA_NO 0 #define CAL_BYS_CTRL2_DUPLICATEDDATA_YES 1 -#define CAL_BYS_CTRL2_FREERUNNING_MASK BIT_MASK(11) +#define CAL_BYS_CTRL2_FREERUNNING_MASK BIT(11) #define CAL_BYS_CTRL2_FREERUNNING_NO 0 #define CAL_BYS_CTRL2_FREERUNNING_YES 1 -#define CAL_RD_DMA_CTRL_GO_MASK BIT_MASK(0) +#define CAL_RD_DMA_CTRL_GO_MASK BIT(0) #define CAL_RD_DMA_CTRL_GO_DIS 0 #define CAL_RD_DMA_CTRL_GO_EN 1 #define CAL_RD_DMA_CTRL_GO_IDLE 0 #define CAL_RD_DMA_CTRL_GO_BUSY 1 -#define CAL_RD_DMA_CTRL_INIT_MASK BIT_MASK(1) +#define CAL_RD_DMA_CTRL_INIT_MASK BIT(1) #define CAL_RD_DMA_CTRL_BW_LIMITER_MASK GENMASK(10, 2) #define CAL_RD_DMA_CTRL_OCP_TAG_CNT_MASK GENMASK(14, 11) #define CAL_RD_DMA_CTRL_PCLK_MASK GENMASK(31, 15) @@ -277,13 +302,13 @@ #define CAL_RD_DMA_CTRL2_CIRC_MODE_SIXTEEN 3 #define CAL_RD_DMA_CTRL2_CIRC_MODE_SIXTYFOUR 4 #define CAL_RD_DMA_CTRL2_CIRC_MODE_RESERVED 5 -#define CAL_RD_DMA_CTRL2_ICM_CSTART_MASK BIT_MASK(3) +#define CAL_RD_DMA_CTRL2_ICM_CSTART_MASK BIT(3) #define CAL_RD_DMA_CTRL2_PATTERN_MASK GENMASK(5, 4) #define CAL_RD_DMA_CTRL2_PATTERN_LINEAR 0 #define CAL_RD_DMA_CTRL2_PATTERN_YUV420 1 #define CAL_RD_DMA_CTRL2_PATTERN_RD2SKIP2 2 #define CAL_RD_DMA_CTRL2_PATTERN_RD2SKIP4 3 -#define CAL_RD_DMA_CTRL2_BYSOUT_LE_WAIT_MASK BIT_MASK(6) +#define CAL_RD_DMA_CTRL2_BYSOUT_LE_WAIT_MASK BIT(6) #define CAL_RD_DMA_CTRL2_BYSOUT_LE_WAIT_FREERUNNING 0 #define CAL_RD_DMA_CTRL2_BYSOUT_LE_WAIT_WAITFORBYSOUT 1 #define CAL_RD_DMA_CTRL2_CIRC_SIZE_MASK GENMASK(29, 16) @@ -300,7 +325,7 @@ #define CAL_WR_DMA_CTRL_PATTERN_WR2SKIP2 2 #define CAL_WR_DMA_CTRL_PATTERN_WR2SKIP4 3 #define CAL_WR_DMA_CTRL_PATTERN_RESERVED 1 -#define CAL_WR_DMA_CTRL_ICM_PSTART_MASK BIT_MASK(5) +#define CAL_WR_DMA_CTRL_ICM_PSTART_MASK BIT(5) #define CAL_WR_DMA_CTRL_DTAG_MASK GENMASK(8, 6) #define CAL_WR_DMA_CTRL_DTAG_ATT_HDR 0 #define CAL_WR_DMA_CTRL_DTAG_ATT_DAT 1 @@ -311,7 +336,7 @@ #define CAL_WR_DMA_CTRL_DTAG_D6 6 #define CAL_WR_DMA_CTRL_DTAG_D7 7 #define CAL_WR_DMA_CTRL_CPORT_MASK GENMASK(13, 9) -#define CAL_WR_DMA_CTRL_STALL_RD_MASK BIT_MASK(14) +#define CAL_WR_DMA_CTRL_STALL_RD_MASK BIT(14) #define CAL_WR_DMA_CTRL_YSIZE_MASK GENMASK(31, 18) #define CAL_WR_DMA_ADDR_MASK GENMASK(31, 4) @@ -327,9 +352,9 @@ #define CAL_WR_DMA_XSIZE_XSKIP_MASK GENMASK(15, 3) #define CAL_WR_DMA_XSIZE_MASK GENMASK(31, 19) -#define CAL_CSI2_PPI_CTRL_IF_EN_MASK BIT_MASK(0) -#define CAL_CSI2_PPI_CTRL_ECC_EN_MASK BIT_MASK(2) -#define CAL_CSI2_PPI_CTRL_FRAME_MASK BIT_MASK(3) +#define CAL_CSI2_PPI_CTRL_IF_EN_MASK BIT(0) +#define CAL_CSI2_PPI_CTRL_ECC_EN_MASK BIT(2) +#define CAL_CSI2_PPI_CTRL_FRAME_MASK BIT(3) #define CAL_CSI2_PPI_CTRL_FRAME_IMMEDIATE 0 #define CAL_CSI2_PPI_CTRL_FRAME 1 @@ -340,18 +365,18 @@ #define CAL_CSI2_COMPLEXIO_CFG_POSITION_2 2 #define CAL_CSI2_COMPLEXIO_CFG_POSITION_1 1 #define CAL_CSI2_COMPLEXIO_CFG_POSITION_NOT_USED 0 -#define CAL_CSI2_COMPLEXIO_CFG_CLOCK_POL_MASK BIT_MASK(3) +#define CAL_CSI2_COMPLEXIO_CFG_CLOCK_POL_MASK BIT(3) #define CAL_CSI2_COMPLEXIO_CFG_POL_PLUSMINUS 0 #define CAL_CSI2_COMPLEXIO_CFG_POL_MINUSPLUS 1 #define CAL_CSI2_COMPLEXIO_CFG_DATA1_POSITION_MASK GENMASK(6, 4) -#define CAL_CSI2_COMPLEXIO_CFG_DATA1_POL_MASK BIT_MASK(7) +#define CAL_CSI2_COMPLEXIO_CFG_DATA1_POL_MASK BIT(7) #define CAL_CSI2_COMPLEXIO_CFG_DATA2_POSITION_MASK GENMASK(10, 8) -#define CAL_CSI2_COMPLEXIO_CFG_DATA2_POL_MASK BIT_MASK(11) +#define CAL_CSI2_COMPLEXIO_CFG_DATA2_POL_MASK BIT(11) #define CAL_CSI2_COMPLEXIO_CFG_DATA3_POSITION_MASK GENMASK(14, 12) -#define CAL_CSI2_COMPLEXIO_CFG_DATA3_POL_MASK BIT_MASK(15) +#define CAL_CSI2_COMPLEXIO_CFG_DATA3_POL_MASK BIT(15) #define CAL_CSI2_COMPLEXIO_CFG_DATA4_POSITION_MASK GENMASK(18, 16) -#define CAL_CSI2_COMPLEXIO_CFG_DATA4_POL_MASK BIT_MASK(19) -#define CAL_CSI2_COMPLEXIO_CFG_PWR_AUTO_MASK BIT_MASK(24) +#define CAL_CSI2_COMPLEXIO_CFG_DATA4_POL_MASK BIT(19) +#define CAL_CSI2_COMPLEXIO_CFG_PWR_AUTO_MASK BIT(24) #define CAL_CSI2_COMPLEXIO_CFG_PWR_STATUS_MASK GENMASK(26, 25) #define CAL_CSI2_COMPLEXIO_CFG_PWR_STATUS_STATE_OFF 0 #define CAL_CSI2_COMPLEXIO_CFG_PWR_STATUS_STATE_ON 1 @@ -360,83 +385,83 @@ #define CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_STATE_OFF 0 #define CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_STATE_ON 1 #define CAL_CSI2_COMPLEXIO_CFG_PWR_CMD_STATE_ULP 2 -#define CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_MASK BIT_MASK(29) +#define CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_MASK BIT(29) #define CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_RESETCOMPLETED 1 #define CAL_CSI2_COMPLEXIO_CFG_RESET_DONE_RESETONGOING 0 -#define CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK BIT_MASK(30) +#define CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_MASK BIT(30) #define CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL 0 #define CAL_CSI2_COMPLEXIO_CFG_RESET_CTRL_OPERATIONAL 1 #define CAL_CSI2_SHORT_PACKET_MASK GENMASK(23, 0) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS1_MASK BIT_MASK(0) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS2_MASK BIT_MASK(1) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS3_MASK BIT_MASK(2) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS4_MASK BIT_MASK(3) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS5_MASK BIT_MASK(4) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS1_MASK BIT_MASK(5) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS2_MASK BIT_MASK(6) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS3_MASK BIT_MASK(7) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS4_MASK BIT_MASK(8) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS5_MASK BIT_MASK(9) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC1_MASK BIT_MASK(10) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC2_MASK BIT_MASK(11) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC3_MASK BIT_MASK(12) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC4_MASK BIT_MASK(13) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC5_MASK BIT_MASK(14) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL1_MASK BIT_MASK(15) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL2_MASK BIT_MASK(16) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL3_MASK BIT_MASK(17) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL4_MASK BIT_MASK(18) -#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL5_MASK BIT_MASK(19) -#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM1_MASK BIT_MASK(20) -#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM2_MASK BIT_MASK(21) -#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM3_MASK BIT_MASK(22) -#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM4_MASK BIT_MASK(23) -#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM5_MASK BIT_MASK(24) -#define CAL_CSI2_COMPLEXIO_IRQ_STATEALLULPMENTER_MASK BIT_MASK(25) -#define CAL_CSI2_COMPLEXIO_IRQ_STATEALLULPMEXIT_MASK BIT_MASK(26) -#define CAL_CSI2_COMPLEXIO_IRQ_FIFO_OVR_MASK BIT_MASK(27) -#define CAL_CSI2_COMPLEXIO_IRQ_SHORT_PACKET_MASK BIT_MASK(28) -#define CAL_CSI2_COMPLEXIO_IRQ_ECC_NO_CORRECTION_MASK BIT_MASK(30) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS1_MASK BIT(0) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS2_MASK BIT(1) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS3_MASK BIT(2) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS4_MASK BIT(3) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTHS5_MASK BIT(4) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS1_MASK BIT(5) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS2_MASK BIT(6) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS3_MASK BIT(7) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS4_MASK BIT(8) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRSOTSYNCHS5_MASK BIT(9) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC1_MASK BIT(10) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC2_MASK BIT(11) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC3_MASK BIT(12) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC4_MASK BIT(13) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRESC5_MASK BIT(14) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL1_MASK BIT(15) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL2_MASK BIT(16) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL3_MASK BIT(17) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL4_MASK BIT(18) +#define CAL_CSI2_COMPLEXIO_IRQ_ERRCONTROL5_MASK BIT(19) +#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM1_MASK BIT(20) +#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM2_MASK BIT(21) +#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM3_MASK BIT(22) +#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM4_MASK BIT(23) +#define CAL_CSI2_COMPLEXIO_IRQ_STATEULPM5_MASK BIT(24) +#define CAL_CSI2_COMPLEXIO_IRQ_STATEALLULPMENTER_MASK BIT(25) +#define CAL_CSI2_COMPLEXIO_IRQ_STATEALLULPMEXIT_MASK BIT(26) +#define CAL_CSI2_COMPLEXIO_IRQ_FIFO_OVR_MASK BIT(27) +#define CAL_CSI2_COMPLEXIO_IRQ_SHORT_PACKET_MASK BIT(28) +#define CAL_CSI2_COMPLEXIO_IRQ_ECC_NO_CORRECTION_MASK BIT(30) #define CAL_CSI2_TIMING_STOP_STATE_COUNTER_IO1_MASK GENMASK(12, 0) -#define CAL_CSI2_TIMING_STOP_STATE_X4_IO1_MASK BIT_MASK(13) -#define CAL_CSI2_TIMING_STOP_STATE_X16_IO1_MASK BIT_MASK(14) -#define CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK BIT_MASK(15) - -#define CAL_CSI2_VC_IRQ_FS_IRQ_0_MASK BIT_MASK(0) -#define CAL_CSI2_VC_IRQ_FE_IRQ_0_MASK BIT_MASK(1) -#define CAL_CSI2_VC_IRQ_LS_IRQ_0_MASK BIT_MASK(2) -#define CAL_CSI2_VC_IRQ_LE_IRQ_0_MASK BIT_MASK(3) -#define CAL_CSI2_VC_IRQ_CS_IRQ_0_MASK BIT_MASK(4) -#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_0_MASK BIT_MASK(5) -#define CAL_CSI2_VC_IRQ_FS_IRQ_1_MASK BIT_MASK(8) -#define CAL_CSI2_VC_IRQ_FE_IRQ_1_MASK BIT_MASK(9) -#define CAL_CSI2_VC_IRQ_LS_IRQ_1_MASK BIT_MASK(10) -#define CAL_CSI2_VC_IRQ_LE_IRQ_1_MASK BIT_MASK(11) -#define CAL_CSI2_VC_IRQ_CS_IRQ_1_MASK BIT_MASK(12) -#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_1_MASK BIT_MASK(13) -#define CAL_CSI2_VC_IRQ_FS_IRQ_2_MASK BIT_MASK(16) -#define CAL_CSI2_VC_IRQ_FE_IRQ_2_MASK BIT_MASK(17) -#define CAL_CSI2_VC_IRQ_LS_IRQ_2_MASK BIT_MASK(18) -#define CAL_CSI2_VC_IRQ_LE_IRQ_2_MASK BIT_MASK(19) -#define CAL_CSI2_VC_IRQ_CS_IRQ_2_MASK BIT_MASK(20) -#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_2_MASK BIT_MASK(21) -#define CAL_CSI2_VC_IRQ_FS_IRQ_3_MASK BIT_MASK(24) -#define CAL_CSI2_VC_IRQ_FE_IRQ_3_MASK BIT_MASK(25) -#define CAL_CSI2_VC_IRQ_LS_IRQ_3_MASK BIT_MASK(26) -#define CAL_CSI2_VC_IRQ_LE_IRQ_3_MASK BIT_MASK(27) -#define CAL_CSI2_VC_IRQ_CS_IRQ_3_MASK BIT_MASK(28) -#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_3_MASK BIT_MASK(29) +#define CAL_CSI2_TIMING_STOP_STATE_X4_IO1_MASK BIT(13) +#define CAL_CSI2_TIMING_STOP_STATE_X16_IO1_MASK BIT(14) +#define CAL_CSI2_TIMING_FORCE_RX_MODE_IO1_MASK BIT(15) + +#define CAL_CSI2_VC_IRQ_FS_IRQ_0_MASK BIT(0) +#define CAL_CSI2_VC_IRQ_FE_IRQ_0_MASK BIT(1) +#define CAL_CSI2_VC_IRQ_LS_IRQ_0_MASK BIT(2) +#define CAL_CSI2_VC_IRQ_LE_IRQ_0_MASK BIT(3) +#define CAL_CSI2_VC_IRQ_CS_IRQ_0_MASK BIT(4) +#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_0_MASK BIT(5) +#define CAL_CSI2_VC_IRQ_FS_IRQ_1_MASK BIT(8) +#define CAL_CSI2_VC_IRQ_FE_IRQ_1_MASK BIT(9) +#define CAL_CSI2_VC_IRQ_LS_IRQ_1_MASK BIT(10) +#define CAL_CSI2_VC_IRQ_LE_IRQ_1_MASK BIT(11) +#define CAL_CSI2_VC_IRQ_CS_IRQ_1_MASK BIT(12) +#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_1_MASK BIT(13) +#define CAL_CSI2_VC_IRQ_FS_IRQ_2_MASK BIT(16) +#define CAL_CSI2_VC_IRQ_FE_IRQ_2_MASK BIT(17) +#define CAL_CSI2_VC_IRQ_LS_IRQ_2_MASK BIT(18) +#define CAL_CSI2_VC_IRQ_LE_IRQ_2_MASK BIT(19) +#define CAL_CSI2_VC_IRQ_CS_IRQ_2_MASK BIT(20) +#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_2_MASK BIT(21) +#define CAL_CSI2_VC_IRQ_FS_IRQ_3_MASK BIT(24) +#define CAL_CSI2_VC_IRQ_FE_IRQ_3_MASK BIT(25) +#define CAL_CSI2_VC_IRQ_LS_IRQ_3_MASK BIT(26) +#define CAL_CSI2_VC_IRQ_LE_IRQ_3_MASK BIT(27) +#define CAL_CSI2_VC_IRQ_CS_IRQ_3_MASK BIT(28) +#define CAL_CSI2_VC_IRQ_ECC_CORRECTION0_IRQ_3_MASK BIT(29) #define CAL_CSI2_CTX_DT_MASK GENMASK(5, 0) #define CAL_CSI2_CTX_VC_MASK GENMASK(7, 6) #define CAL_CSI2_CTX_CPORT_MASK GENMASK(12, 8) -#define CAL_CSI2_CTX_ATT_MASK BIT_MASK(13) +#define CAL_CSI2_CTX_ATT_MASK BIT(13) #define CAL_CSI2_CTX_ATT_PIX 0 #define CAL_CSI2_CTX_ATT 1 -#define CAL_CSI2_CTX_PACK_MODE_MASK BIT_MASK(14) +#define CAL_CSI2_CTX_PACK_MODE_MASK BIT(14) #define CAL_CSI2_CTX_PACK_MODE_LINE 0 #define CAL_CSI2_CTX_PACK_MODE_FRAME 1 #define CAL_CSI2_CTX_LINES_MASK GENMASK(29, 16) @@ -445,7 +470,7 @@ #define CAL_CSI2_PHY_REG0_THS_SETTLE_MASK GENMASK(7, 0) #define CAL_CSI2_PHY_REG0_THS_TERM_MASK GENMASK(15, 8) -#define CAL_CSI2_PHY_REG0_HSCLOCKCONFIG_MASK BIT_MASK(24) +#define CAL_CSI2_PHY_REG0_HSCLOCKCONFIG_MASK BIT(24) #define CAL_CSI2_PHY_REG0_HSCLOCKCONFIG_DISABLE 1 #define CAL_CSI2_PHY_REG0_HSCLOCKCONFIG_ENABLE 0 @@ -453,24 +478,26 @@ #define CAL_CSI2_PHY_REG1_CTRLCLK_DIV_FACTOR_MASK GENMASK(9, 8) #define CAL_CSI2_PHY_REG1_DPHY_HS_SYNC_PATTERN_MASK GENMASK(17, 10) #define CAL_CSI2_PHY_REG1_TCLK_TERM_MASK GENMASK(24, 18) -#define CAL_CSI2_PHY_REG1_CLOCK_MISS_DETECTOR_STATUS_MASK BIT_MASK(25) +#define CAL_CSI2_PHY_REG1_CLOCK_MISS_DETECTOR_STATUS_MASK BIT(25) #define CAL_CSI2_PHY_REG1_CLOCK_MISS_DETECTOR_STATUS_ERROR 1 #define CAL_CSI2_PHY_REG1_CLOCK_MISS_DETECTOR_STATUS_SUCCESS 0 #define CAL_CSI2_PHY_REG1_RESET_DONE_STATUS_MASK GENMASK(29, 28) +#define CAL_CSI2_PHY_REG10_I933_LDO_DISABLE_MASK BIT(6) + #define CAL_CSI2_PHY_REG2_CCP2_SYNC_PATTERN_MASK GENMASK(23, 0) #define CAL_CSI2_PHY_REG2_TRIGGER_CMD_RXTRIGESC3_MASK GENMASK(25, 24) #define CAL_CSI2_PHY_REG2_TRIGGER_CMD_RXTRIGESC2_MASK GENMASK(27, 26) #define CAL_CSI2_PHY_REG2_TRIGGER_CMD_RXTRIGESC1_MASK GENMASK(29, 28) #define CAL_CSI2_PHY_REG2_TRIGGER_CMD_RXTRIGESC0_MASK GENMASK(31, 30) -#define CM_CAMERRX_CTRL_CSI1_CTRLCLKEN_MASK BIT_MASK(0) +#define CM_CAMERRX_CTRL_CSI1_CTRLCLKEN_MASK BIT(0) #define CM_CAMERRX_CTRL_CSI1_CAMMODE_MASK GENMASK(2, 1) #define CM_CAMERRX_CTRL_CSI1_LANEENABLE_MASK GENMASK(4, 3) -#define CM_CAMERRX_CTRL_CSI1_MODE_MASK BIT_MASK(5) -#define CM_CAMERRX_CTRL_CSI0_CTRLCLKEN_MASK BIT_MASK(10) +#define CM_CAMERRX_CTRL_CSI1_MODE_MASK BIT(5) +#define CM_CAMERRX_CTRL_CSI0_CTRLCLKEN_MASK BIT(10) #define CM_CAMERRX_CTRL_CSI0_CAMMODE_MASK GENMASK(12, 11) #define CM_CAMERRX_CTRL_CSI0_LANEENABLE_MASK GENMASK(16, 13) -#define CM_CAMERRX_CTRL_CSI0_MODE_MASK BIT_MASK(17) +#define CM_CAMERRX_CTRL_CSI0_MODE_MASK BIT(17) #endif diff --git a/drivers/media/platform/ti-vpe/csc.c b/drivers/media/platform/ti-vpe/csc.c index 834114a4eebe..f4e0cf72d1cf 100644 --- a/drivers/media/platform/ti-vpe/csc.c +++ b/drivers/media/platform/ti-vpe/csc.c @@ -149,36 +149,28 @@ void csc_set_coeff(struct csc_data *csc, u32 *csc_reg0, enum v4l2_quantization src_quantization, dst_quantization; u32 src_pixelformat, dst_pixelformat; - switch (src_fmt->type) { - case V4L2_BUF_TYPE_VIDEO_OUTPUT: - pix = &src_fmt->fmt.pix; - src_pixelformat = pix->pixelformat; - src_ycbcr_enc = pix->ycbcr_enc; - src_quantization = pix->quantization; - break; - case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: - default: + if (V4L2_TYPE_IS_MULTIPLANAR(src_fmt->type)) { mp = &src_fmt->fmt.pix_mp; src_pixelformat = mp->pixelformat; src_ycbcr_enc = mp->ycbcr_enc; src_quantization = mp->quantization; - break; + } else { + pix = &src_fmt->fmt.pix; + src_pixelformat = pix->pixelformat; + src_ycbcr_enc = pix->ycbcr_enc; + src_quantization = pix->quantization; } - switch (dst_fmt->type) { - case V4L2_BUF_TYPE_VIDEO_OUTPUT: - pix = &dst_fmt->fmt.pix; - dst_pixelformat = pix->pixelformat; - dst_ycbcr_enc = pix->ycbcr_enc; - dst_quantization = pix->quantization; - break; - case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: - default: + if (V4L2_TYPE_IS_MULTIPLANAR(dst_fmt->type)) { mp = &dst_fmt->fmt.pix_mp; dst_pixelformat = mp->pixelformat; dst_ycbcr_enc = mp->ycbcr_enc; dst_quantization = mp->quantization; - break; + } else { + pix = &dst_fmt->fmt.pix; + dst_pixelformat = pix->pixelformat; + dst_ycbcr_enc = pix->ycbcr_enc; + dst_quantization = pix->quantization; } src_finfo = v4l2_format_info(src_pixelformat); diff --git a/drivers/media/platform/vimc/vimc-scaler.c b/drivers/media/platform/vimc/vimc-scaler.c index 2f88a7d9d67b..e2e551bc3ded 100644 --- a/drivers/media/platform/vimc/vimc-scaler.c +++ b/drivers/media/platform/vimc/vimc-scaler.c @@ -8,6 +8,7 @@ #include <linux/moduleparam.h> #include <linux/vmalloc.h> #include <linux/v4l2-mediabus.h> +#include <media/v4l2-rect.h> #include <media/v4l2-subdev.h> #include "vimc-common.h" @@ -18,6 +19,9 @@ MODULE_PARM_DESC(sca_mult, " the image size multiplier"); #define MAX_ZOOM 8 +#define VIMC_SCA_FMT_WIDTH_DEFAULT 640 +#define VIMC_SCA_FMT_HEIGHT_DEFAULT 480 + struct vimc_sca_device { struct vimc_ent_device ved; struct v4l2_subdev sd; @@ -25,6 +29,7 @@ struct vimc_sca_device { * with the width and hight multiplied by mult */ struct v4l2_mbus_framefmt sink_fmt; + struct v4l2_rect crop_rect; /* Values calculated when the stream starts */ u8 *src_frame; unsigned int src_line_size; @@ -33,22 +38,64 @@ struct vimc_sca_device { }; static const struct v4l2_mbus_framefmt sink_fmt_default = { - .width = 640, - .height = 480, + .width = VIMC_SCA_FMT_WIDTH_DEFAULT, + .height = VIMC_SCA_FMT_HEIGHT_DEFAULT, .code = MEDIA_BUS_FMT_RGB888_1X24, .field = V4L2_FIELD_NONE, .colorspace = V4L2_COLORSPACE_DEFAULT, }; +static const struct v4l2_rect crop_rect_default = { + .width = VIMC_SCA_FMT_WIDTH_DEFAULT, + .height = VIMC_SCA_FMT_HEIGHT_DEFAULT, + .top = 0, + .left = 0, +}; + +static const struct v4l2_rect crop_rect_min = { + .width = VIMC_FRAME_MIN_WIDTH, + .height = VIMC_FRAME_MIN_HEIGHT, + .top = 0, + .left = 0, +}; + +static struct v4l2_rect +vimc_sca_get_crop_bound_sink(const struct v4l2_mbus_framefmt *sink_fmt) +{ + /* Get the crop bounds to clamp the crop rectangle correctly */ + struct v4l2_rect r = { + .left = 0, + .top = 0, + .width = sink_fmt->width, + .height = sink_fmt->height, + }; + return r; +} + +static void vimc_sca_adjust_sink_crop(struct v4l2_rect *r, + const struct v4l2_mbus_framefmt *sink_fmt) +{ + const struct v4l2_rect sink_rect = + vimc_sca_get_crop_bound_sink(sink_fmt); + + /* Disallow rectangles smaller than the minimal one. */ + v4l2_rect_set_min_size(r, &crop_rect_min); + v4l2_rect_map_inside(r, &sink_rect); +} + static int vimc_sca_init_cfg(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg) { struct v4l2_mbus_framefmt *mf; + struct v4l2_rect *r; unsigned int i; mf = v4l2_subdev_get_try_format(sd, cfg, 0); *mf = sink_fmt_default; + r = v4l2_subdev_get_try_crop(sd, cfg, 0); + *r = crop_rect_default; + for (i = 1; i < sd->entity.num_pads; i++) { mf = v4l2_subdev_get_try_format(sd, cfg, i); *mf = sink_fmt_default; @@ -107,16 +154,21 @@ static int vimc_sca_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_format *format) { struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); + struct v4l2_rect *crop_rect; /* Get the current sink format */ - format->format = (format->which == V4L2_SUBDEV_FORMAT_TRY) ? - *v4l2_subdev_get_try_format(sd, cfg, 0) : - vsca->sink_fmt; + if (format->which == V4L2_SUBDEV_FORMAT_TRY) { + format->format = *v4l2_subdev_get_try_format(sd, cfg, 0); + crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); + } else { + format->format = vsca->sink_fmt; + crop_rect = &vsca->crop_rect; + } /* Scale the frame size for the source pad */ if (VIMC_IS_SRC(format->pad)) { - format->format.width = vsca->sink_fmt.width * sca_mult; - format->format.height = vsca->sink_fmt.height * sca_mult; + format->format.width = crop_rect->width * sca_mult; + format->format.height = crop_rect->height * sca_mult; } return 0; @@ -148,6 +200,7 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd, { struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *sink_fmt; + struct v4l2_rect *crop_rect; if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) { /* Do not change the format while stream is on */ @@ -155,8 +208,10 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd, return -EBUSY; sink_fmt = &vsca->sink_fmt; + crop_rect = &vsca->crop_rect; } else { sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); } /* @@ -165,8 +220,8 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd, */ if (VIMC_IS_SRC(fmt->pad)) { fmt->format = *sink_fmt; - fmt->format.width = sink_fmt->width * sca_mult; - fmt->format.height = sink_fmt->height * sca_mult; + fmt->format.width = crop_rect->width * sca_mult; + fmt->format.height = crop_rect->height * sca_mult; } else { /* Set the new format in the sink pad */ vimc_sca_adjust_sink_fmt(&fmt->format); @@ -184,6 +239,78 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd, fmt->format.xfer_func, fmt->format.ycbcr_enc); *sink_fmt = fmt->format; + + /* Do the crop, but respect the current bounds */ + vimc_sca_adjust_sink_crop(crop_rect, sink_fmt); + } + + return 0; +} + +static int vimc_sca_get_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_selection *sel) +{ + struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); + struct v4l2_mbus_framefmt *sink_fmt; + struct v4l2_rect *crop_rect; + + if (VIMC_IS_SRC(sel->pad)) + return -EINVAL; + + if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) { + sink_fmt = &vsca->sink_fmt; + crop_rect = &vsca->crop_rect; + } else { + sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); + } + + switch (sel->target) { + case V4L2_SEL_TGT_CROP: + sel->r = *crop_rect; + break; + case V4L2_SEL_TGT_CROP_BOUNDS: + sel->r = vimc_sca_get_crop_bound_sink(sink_fmt); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int vimc_sca_set_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_selection *sel) +{ + struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); + struct v4l2_mbus_framefmt *sink_fmt; + struct v4l2_rect *crop_rect; + + if (VIMC_IS_SRC(sel->pad)) + return -EINVAL; + + if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) { + /* Do not change the format while stream is on */ + if (vsca->src_frame) + return -EBUSY; + + crop_rect = &vsca->crop_rect; + sink_fmt = &vsca->sink_fmt; + } else { + crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); + sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + } + + switch (sel->target) { + case V4L2_SEL_TGT_CROP: + /* Do the crop, but respect the current bounds */ + vimc_sca_adjust_sink_crop(&sel->r, sink_fmt); + *crop_rect = sel->r; + break; + default: + return -EINVAL; } return 0; @@ -195,6 +322,8 @@ static const struct v4l2_subdev_pad_ops vimc_sca_pad_ops = { .enum_frame_size = vimc_sca_enum_frame_size, .get_fmt = vimc_sca_get_fmt, .set_fmt = vimc_sca_set_fmt, + .get_selection = vimc_sca_get_selection, + .set_selection = vimc_sca_set_selection, }; static int vimc_sca_s_stream(struct v4l2_subdev *sd, int enable) @@ -213,11 +342,11 @@ static int vimc_sca_s_stream(struct v4l2_subdev *sd, int enable) vsca->bpp = vpix->bpp; /* Calculate the width in bytes of the src frame */ - vsca->src_line_size = vsca->sink_fmt.width * + vsca->src_line_size = vsca->crop_rect.width * sca_mult * vsca->bpp; /* Calculate the frame size of the source pad */ - frame_size = vsca->src_line_size * vsca->sink_fmt.height * + frame_size = vsca->src_line_size * vsca->crop_rect.height * sca_mult; /* Allocate the frame buffer. Use vmalloc to be able to @@ -259,9 +388,10 @@ static void vimc_sca_fill_pix(u8 *const ptr, } static void vimc_sca_scale_pix(const struct vimc_sca_device *const vsca, - const unsigned int lin, const unsigned int col, + unsigned int lin, unsigned int col, const u8 *const sink_frame) { + const struct v4l2_rect crop_rect = vsca->crop_rect; unsigned int i, j, index; const u8 *pixel; @@ -278,8 +408,10 @@ static void vimc_sca_scale_pix(const struct vimc_sca_device *const vsca, /* point to the place we are going to put the first pixel * in the scaled src frame */ + lin -= crop_rect.top; + col -= crop_rect.left; index = VIMC_FRAME_INDEX(lin * sca_mult, col * sca_mult, - vsca->sink_fmt.width * sca_mult, vsca->bpp); + crop_rect.width * sca_mult, vsca->bpp); dev_dbg(vsca->ved.dev, "sca: %s: scale_pix src pos %dx%d, index %d\n", vsca->sd.name, lin * sca_mult, col * sca_mult, index); @@ -307,12 +439,13 @@ static void vimc_sca_scale_pix(const struct vimc_sca_device *const vsca, static void vimc_sca_fill_src_frame(const struct vimc_sca_device *const vsca, const u8 *const sink_frame) { + const struct v4l2_rect r = vsca->crop_rect; unsigned int i, j; /* Scale each pixel from the original sink frame */ /* TODO: implement scale down, only scale up is supported for now */ - for (i = 0; i < vsca->sink_fmt.height; i++) - for (j = 0; j < vsca->sink_fmt.width; j++) + for (i = r.top; i < r.top + r.height; i++) + for (j = r.left; j < r.left + r.width; j++) vimc_sca_scale_pix(vsca, i, j, sink_frame); } @@ -384,5 +517,8 @@ struct vimc_ent_device *vimc_sca_add(struct vimc_device *vimc, /* Initialize the frame format */ vsca->sink_fmt = sink_fmt_default; + /* Initialize the crop selection */ + vsca->crop_rect = crop_rect_default; + return &vsca->ved; } diff --git a/drivers/media/platform/vivid/Makefile b/drivers/media/platform/vivid/Makefile index e8a50c506dc9..b12ad0152a3e 100644 --- a/drivers/media/platform/vivid/Makefile +++ b/drivers/media/platform/vivid/Makefile @@ -3,7 +3,8 @@ vivid-objs := vivid-core.o vivid-ctrls.o vivid-vid-common.o vivid-vbi-gen.o \ vivid-vid-cap.o vivid-vid-out.o vivid-kthread-cap.o vivid-kthread-out.o \ vivid-radio-rx.o vivid-radio-tx.o vivid-radio-common.o \ vivid-rds-gen.o vivid-sdr-cap.o vivid-vbi-cap.o vivid-vbi-out.o \ - vivid-osd.o vivid-meta-cap.o vivid-meta-out.o + vivid-osd.o vivid-meta-cap.o vivid-meta-out.o \ + vivid-kthread-touch.o vivid-touch-cap.o ifeq ($(CONFIG_VIDEO_VIVID_CEC),y) vivid-objs += vivid-cec.o endif diff --git a/drivers/media/platform/vivid/vivid-core.c b/drivers/media/platform/vivid/vivid-core.c index c184f9b0be69..15091cbf6de7 100644 --- a/drivers/media/platform/vivid/vivid-core.c +++ b/drivers/media/platform/vivid/vivid-core.c @@ -39,6 +39,7 @@ #include "vivid-ctrls.h" #include "vivid-meta-cap.h" #include "vivid-meta-out.h" +#include "vivid-touch-cap.h" #define VIVID_MODULE_NAME "vivid" @@ -89,6 +90,10 @@ static int meta_out_nr[VIVID_MAX_DEVS] = { [0 ... (VIVID_MAX_DEVS - 1)] = -1 }; module_param_array(meta_out_nr, int, NULL, 0444); MODULE_PARM_DESC(meta_out_nr, " videoX start number, -1 is autodetect"); +static int touch_cap_nr[VIVID_MAX_DEVS] = { [0 ... (VIVID_MAX_DEVS - 1)] = -1 }; +module_param_array(touch_cap_nr, int, NULL, 0444); +MODULE_PARM_DESC(touch_cap_nr, " v4l-touchX start number, -1 is autodetect"); + static int ccs_cap_mode[VIVID_MAX_DEVS] = { [0 ... (VIVID_MAX_DEVS - 1)] = -1 }; module_param_array(ccs_cap_mode, int, NULL, 0444); MODULE_PARM_DESC(ccs_cap_mode, " capture crop/compose/scale mode:\n" @@ -110,10 +115,10 @@ MODULE_PARM_DESC(multiplanar, " 1 (default) creates a single planar device, 2 cr * vbi-out + vid-out + meta-cap */ static unsigned int node_types[VIVID_MAX_DEVS] = { - [0 ... (VIVID_MAX_DEVS - 1)] = 0x61d3d + [0 ... (VIVID_MAX_DEVS - 1)] = 0xe1d3d }; module_param_array(node_types, uint, NULL, 0444); -MODULE_PARM_DESC(node_types, " node types, default is 0x61d3d. Bitmask with the following meaning:\n" +MODULE_PARM_DESC(node_types, " node types, default is 0xe1d3d. Bitmask with the following meaning:\n" "\t\t bit 0: Video Capture node\n" "\t\t bit 2-3: VBI Capture node: 0 = none, 1 = raw vbi, 2 = sliced vbi, 3 = both\n" "\t\t bit 4: Radio Receiver node\n" @@ -123,7 +128,8 @@ MODULE_PARM_DESC(node_types, " node types, default is 0x61d3d. Bitmask with the "\t\t bit 12: Radio Transmitter node\n" "\t\t bit 16: Framebuffer for testing overlays\n" "\t\t bit 17: Metadata Capture node\n" - "\t\t bit 18: Metadata Output node\n"); + "\t\t bit 18: Metadata Output node\n" + "\t\t bit 19: Touch Capture node\n"); /* Default: 4 inputs */ static unsigned num_inputs[VIVID_MAX_DEVS] = { [0 ... (VIVID_MAX_DEVS - 1)] = 4 }; @@ -223,7 +229,8 @@ static int vidioc_querycap(struct file *file, void *priv, dev->vbi_cap_caps | dev->vbi_out_caps | dev->radio_rx_caps | dev->radio_tx_caps | dev->sdr_cap_caps | dev->meta_cap_caps | - dev->meta_out_caps | V4L2_CAP_DEVICE_CAPS; + dev->meta_out_caps | dev->touch_cap_caps | + V4L2_CAP_DEVICE_CAPS; return 0; } @@ -377,6 +384,8 @@ static int vidioc_g_parm(struct file *file, void *fh, { struct video_device *vdev = video_devdata(file); + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_parm_tch(file, fh, parm); if (vdev->vfl_dir == VFL_DIR_RX) return vivid_vid_cap_g_parm(file, fh, parm); return vivid_vid_out_g_parm(file, fh, parm); @@ -432,6 +441,104 @@ static __poll_t vivid_radio_poll(struct file *file, struct poll_table_struct *wa return vivid_radio_tx_poll(file, wait); } +static int vivid_enum_input(struct file *file, void *priv, + struct v4l2_input *inp) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_enum_input_tch(file, priv, inp); + return vidioc_enum_input(file, priv, inp); +} + +static int vivid_g_input(struct file *file, void *priv, unsigned int *i) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_input_tch(file, priv, i); + return vidioc_g_input(file, priv, i); +} + +static int vivid_s_input(struct file *file, void *priv, unsigned int i) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_s_input_tch(file, priv, i); + return vidioc_s_input(file, priv, i); +} + +static int vivid_enum_fmt_cap(struct file *file, void *priv, + struct v4l2_fmtdesc *f) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_enum_fmt_tch(file, priv, f); + return vivid_enum_fmt_vid(file, priv, f); +} + +static int vivid_g_fmt_cap(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_fmt_tch(file, priv, f); + return vidioc_g_fmt_vid_cap(file, priv, f); +} + +static int vivid_try_fmt_cap(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_fmt_tch(file, priv, f); + return vidioc_try_fmt_vid_cap(file, priv, f); +} + +static int vivid_s_fmt_cap(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_fmt_tch(file, priv, f); + return vidioc_s_fmt_vid_cap(file, priv, f); +} + +static int vivid_g_fmt_cap_mplane(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_fmt_tch_mplane(file, priv, f); + return vidioc_g_fmt_vid_cap_mplane(file, priv, f); +} + +static int vivid_try_fmt_cap_mplane(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_fmt_tch_mplane(file, priv, f); + return vidioc_try_fmt_vid_cap_mplane(file, priv, f); +} + +static int vivid_s_fmt_cap_mplane(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct video_device *vdev = video_devdata(file); + + if (vdev->vfl_type == VFL_TYPE_TOUCH) + return vivid_g_fmt_tch_mplane(file, priv, f); + return vidioc_s_fmt_vid_cap_mplane(file, priv, f); +} + static bool vivid_is_in_use(struct video_device *vdev) { unsigned long flags; @@ -453,7 +560,8 @@ static bool vivid_is_last_user(struct vivid_dev *dev) vivid_is_in_use(&dev->radio_rx_dev) + vivid_is_in_use(&dev->radio_tx_dev) + vivid_is_in_use(&dev->meta_cap_dev) + - vivid_is_in_use(&dev->meta_out_dev); + vivid_is_in_use(&dev->meta_out_dev) + + vivid_is_in_use(&dev->touch_cap_dev); return uses == 1; } @@ -481,6 +589,7 @@ static int vivid_fop_release(struct file *file) set_bit(V4L2_FL_REGISTERED, &dev->radio_tx_dev.flags); set_bit(V4L2_FL_REGISTERED, &dev->meta_cap_dev.flags); set_bit(V4L2_FL_REGISTERED, &dev->meta_out_dev.flags); + set_bit(V4L2_FL_REGISTERED, &dev->touch_cap_dev.flags); } mutex_unlock(&dev->mutex); if (file->private_data == dev->overlay_cap_owner) @@ -522,13 +631,13 @@ static const struct v4l2_file_operations vivid_radio_fops = { static const struct v4l2_ioctl_ops vivid_ioctl_ops = { .vidioc_querycap = vidioc_querycap, - .vidioc_enum_fmt_vid_cap = vivid_enum_fmt_vid, - .vidioc_g_fmt_vid_cap = vidioc_g_fmt_vid_cap, - .vidioc_try_fmt_vid_cap = vidioc_try_fmt_vid_cap, - .vidioc_s_fmt_vid_cap = vidioc_s_fmt_vid_cap, - .vidioc_g_fmt_vid_cap_mplane = vidioc_g_fmt_vid_cap_mplane, - .vidioc_try_fmt_vid_cap_mplane = vidioc_try_fmt_vid_cap_mplane, - .vidioc_s_fmt_vid_cap_mplane = vidioc_s_fmt_vid_cap_mplane, + .vidioc_enum_fmt_vid_cap = vivid_enum_fmt_cap, + .vidioc_g_fmt_vid_cap = vivid_g_fmt_cap, + .vidioc_try_fmt_vid_cap = vivid_try_fmt_cap, + .vidioc_s_fmt_vid_cap = vivid_s_fmt_cap, + .vidioc_g_fmt_vid_cap_mplane = vivid_g_fmt_cap_mplane, + .vidioc_try_fmt_vid_cap_mplane = vivid_try_fmt_cap_mplane, + .vidioc_s_fmt_vid_cap_mplane = vivid_s_fmt_cap_mplane, .vidioc_enum_fmt_vid_out = vivid_enum_fmt_vid, .vidioc_g_fmt_vid_out = vidioc_g_fmt_vid_out, @@ -590,9 +699,9 @@ static const struct v4l2_ioctl_ops vivid_ioctl_ops = { .vidioc_streamon = vb2_ioctl_streamon, .vidioc_streamoff = vb2_ioctl_streamoff, - .vidioc_enum_input = vidioc_enum_input, - .vidioc_g_input = vidioc_g_input, - .vidioc_s_input = vidioc_s_input, + .vidioc_enum_input = vivid_enum_input, + .vidioc_g_input = vivid_g_input, + .vidioc_s_input = vivid_s_input, .vidioc_s_audio = vidioc_s_audio, .vidioc_g_audio = vidioc_g_audio, .vidioc_enumaudio = vidioc_enumaudio, @@ -921,6 +1030,9 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) dev->has_scaler_out ? 'Y' : 'N'); } + /* do we create a touch capture device */ + dev->has_touch_cap = node_type & 0x80000; + /* end detecting feature set */ if (dev->has_vid_cap) { @@ -994,6 +1106,13 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) if (dev->has_audio_outputs) dev->meta_out_caps |= V4L2_CAP_AUDIO; } + /* set up the capabilities of the touch capture device */ + if (dev->has_touch_cap) { + dev->touch_cap_caps = V4L2_CAP_TOUCH | V4L2_CAP_STREAMING | + V4L2_CAP_READWRITE; + dev->touch_cap_caps |= dev->multiplanar ? + V4L2_CAP_VIDEO_CAPTURE_MPLANE : V4L2_CAP_VIDEO_CAPTURE; + } ret = -ENOMEM; /* initialize the test pattern generator */ @@ -1129,6 +1248,9 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) v4l2_disable_ioctl(&dev->vbi_out_dev, VIDIOC_G_FREQUENCY); v4l2_disable_ioctl(&dev->meta_out_dev, VIDIOC_S_FREQUENCY); v4l2_disable_ioctl(&dev->meta_out_dev, VIDIOC_G_FREQUENCY); + v4l2_disable_ioctl(&dev->touch_cap_dev, VIDIOC_S_PARM); + v4l2_disable_ioctl(&dev->touch_cap_dev, VIDIOC_ENUM_FRAMESIZES); + v4l2_disable_ioctl(&dev->touch_cap_dev, VIDIOC_ENUM_FRAMEINTERVALS); /* configure internal data */ dev->fmt_cap = &vivid_formats[0]; @@ -1201,6 +1323,11 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) dev->fb_cap.fmt.bytesperline = dev->src_rect.width * tpg_g_twopixelsize(&dev->tpg, 0) / 2; dev->fb_cap.fmt.sizeimage = dev->src_rect.height * dev->fb_cap.fmt.bytesperline; + /* update touch configuration */ + dev->timeperframe_tch_cap.numerator = 1; + dev->timeperframe_tch_cap.denominator = 10; + vivid_set_touch(dev, 0); + /* initialize locks */ spin_lock_init(&dev->slock); mutex_init(&dev->mutex); @@ -1213,6 +1340,7 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) INIT_LIST_HEAD(&dev->sdr_cap_active); INIT_LIST_HEAD(&dev->meta_cap_active); INIT_LIST_HEAD(&dev->meta_out_active); + INIT_LIST_HEAD(&dev->touch_cap_active); INIT_LIST_HEAD(&dev->cec_work_list); spin_lock_init(&dev->cec_slock); @@ -1294,6 +1422,15 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) goto unreg_dev; } + if (dev->has_touch_cap) { + /* initialize touch_cap queue */ + ret = vivid_create_queue(dev, &dev->vb_touch_cap_q, + V4L2_BUF_TYPE_VIDEO_CAPTURE, 1, + &vivid_touch_cap_qops); + if (ret) + goto unreg_dev; + } + if (dev->has_fb) { /* Create framebuffer for testing capture/output overlay */ ret = vivid_fb_init(dev); @@ -1345,6 +1482,7 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) v4l2_ctrl_handler_setup(&dev->ctrl_hdl_sdr_cap); v4l2_ctrl_handler_setup(&dev->ctrl_hdl_meta_cap); v4l2_ctrl_handler_setup(&dev->ctrl_hdl_meta_out); + v4l2_ctrl_handler_setup(&dev->ctrl_hdl_touch_cap); /* finally start creating the device nodes */ if (dev->has_vid_cap) { @@ -1635,6 +1773,35 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) video_device_node_name(vfd)); } + if (dev->has_touch_cap) { + vfd = &dev->touch_cap_dev; + snprintf(vfd->name, sizeof(vfd->name), + "vivid-%03d-touch-cap", inst); + vfd->fops = &vivid_fops; + vfd->ioctl_ops = &vivid_ioctl_ops; + vfd->device_caps = dev->touch_cap_caps; + vfd->release = video_device_release_empty; + vfd->v4l2_dev = &dev->v4l2_dev; + vfd->queue = &dev->vb_touch_cap_q; + vfd->tvnorms = tvnorms_cap; + vfd->lock = &dev->mutex; + video_set_drvdata(vfd, dev); +#ifdef CONFIG_MEDIA_CONTROLLER + dev->touch_cap_pad.flags = MEDIA_PAD_FL_SINK; + ret = media_entity_pads_init(&vfd->entity, 1, + &dev->touch_cap_pad); + if (ret) + goto unreg_dev; +#endif + ret = video_register_device(vfd, VFL_TYPE_TOUCH, + touch_cap_nr[inst]); + if (ret < 0) + goto unreg_dev; + v4l2_info(&dev->v4l2_dev, + "V4L2 touch capture device registered as %s\n", + video_device_node_name(vfd)); + } + #ifdef CONFIG_MEDIA_CONTROLLER /* Register the media device */ ret = media_device_register(&dev->mdev); @@ -1651,6 +1818,7 @@ static int vivid_create_instance(struct platform_device *pdev, int inst) return 0; unreg_dev: + video_unregister_device(&dev->touch_cap_dev); video_unregister_device(&dev->meta_out_dev); video_unregister_device(&dev->meta_cap_dev); video_unregister_device(&dev->radio_tx_dev); @@ -1778,6 +1946,11 @@ static int vivid_remove(struct platform_device *pdev) video_device_node_name(&dev->meta_out_dev)); video_unregister_device(&dev->meta_out_dev); } + if (dev->has_touch_cap) { + v4l2_info(&dev->v4l2_dev, "unregistering %s\n", + video_device_node_name(&dev->touch_cap_dev)); + video_unregister_device(&dev->touch_cap_dev); + } cec_unregister_adapter(dev->cec_rx_adap); for (j = 0; j < MAX_OUTPUTS; j++) cec_unregister_adapter(dev->cec_tx_adap[j]); diff --git a/drivers/media/platform/vivid/vivid-core.h b/drivers/media/platform/vivid/vivid-core.h index 59192b67231c..99e69b8f770f 100644 --- a/drivers/media/platform/vivid/vivid-core.h +++ b/drivers/media/platform/vivid/vivid-core.h @@ -133,6 +133,7 @@ struct vivid_dev { struct media_pad sdr_cap_pad; struct media_pad meta_cap_pad; struct media_pad meta_out_pad; + struct media_pad touch_cap_pad; #endif struct v4l2_ctrl_handler ctrl_hdl_user_gen; struct v4l2_ctrl_handler ctrl_hdl_user_vid; @@ -159,6 +160,8 @@ struct vivid_dev { struct v4l2_ctrl_handler ctrl_hdl_meta_cap; struct video_device meta_out_dev; struct v4l2_ctrl_handler ctrl_hdl_meta_out; + struct video_device touch_cap_dev; + struct v4l2_ctrl_handler ctrl_hdl_touch_cap; spinlock_t slock; struct mutex mutex; @@ -173,6 +176,7 @@ struct vivid_dev { u32 radio_tx_caps; u32 meta_cap_caps; u32 meta_out_caps; + u32 touch_cap_caps; /* supported features */ bool multiplanar; @@ -201,6 +205,7 @@ struct vivid_dev { bool has_meta_cap; bool has_meta_out; bool has_tv_tuner; + bool has_touch_cap; bool can_loop_video; @@ -404,6 +409,8 @@ struct vivid_dev { struct list_head vbi_cap_active; struct vb2_queue vb_meta_cap_q; struct list_head meta_cap_active; + struct vb2_queue vb_touch_cap_q; + struct list_head touch_cap_active; /* thread for generating video capture stream */ struct task_struct *kthread_vid_cap; @@ -425,6 +432,19 @@ struct vivid_dev { u32 meta_cap_seq_count; bool meta_cap_streaming; + /* Touch capture */ + struct task_struct *kthread_touch_cap; + unsigned long jiffies_touch_cap; + u64 touch_cap_stream_start; + u32 touch_cap_seq_offset; + bool touch_cap_seq_resync; + u32 touch_cap_seq_start; + u32 touch_cap_seq_count; + bool touch_cap_streaming; + struct v4l2_fract timeperframe_tch_cap; + struct v4l2_pix_format tch_format; + int tch_pat_random; + /* video output */ const struct vivid_fmt *fmt_out; struct v4l2_fract timeperframe_vid_out; diff --git a/drivers/media/platform/vivid/vivid-ctrls.c b/drivers/media/platform/vivid/vivid-ctrls.c index 68e8124c7973..334130568dcb 100644 --- a/drivers/media/platform/vivid/vivid-ctrls.c +++ b/drivers/media/platform/vivid/vivid-ctrls.c @@ -1508,6 +1508,7 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap, struct v4l2_ctrl_handler *hdl_sdr_cap = &dev->ctrl_hdl_sdr_cap; struct v4l2_ctrl_handler *hdl_meta_cap = &dev->ctrl_hdl_meta_cap; struct v4l2_ctrl_handler *hdl_meta_out = &dev->ctrl_hdl_meta_out; + struct v4l2_ctrl_handler *hdl_tch_cap = &dev->ctrl_hdl_touch_cap; struct v4l2_ctrl_config vivid_ctrl_dv_timings = { .ops = &vivid_vid_cap_ctrl_ops, @@ -1551,6 +1552,8 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap, v4l2_ctrl_new_custom(hdl_meta_cap, &vivid_ctrl_class, NULL); v4l2_ctrl_handler_init(hdl_meta_out, 2); v4l2_ctrl_new_custom(hdl_meta_out, &vivid_ctrl_class, NULL); + v4l2_ctrl_handler_init(hdl_tch_cap, 2); + v4l2_ctrl_new_custom(hdl_tch_cap, &vivid_ctrl_class, NULL); /* User Controls */ dev->volume = v4l2_ctrl_new_std(hdl_user_aud, NULL, @@ -1904,6 +1907,13 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap, return hdl_meta_out->error; dev->meta_out_dev.ctrl_handler = hdl_meta_out; } + if (dev->has_touch_cap) { + v4l2_ctrl_add_handler(hdl_tch_cap, hdl_user_gen, NULL, false); + v4l2_ctrl_add_handler(hdl_tch_cap, hdl_streaming, NULL, false); + if (hdl_tch_cap->error) + return hdl_tch_cap->error; + dev->touch_cap_dev.ctrl_handler = hdl_tch_cap; + } return 0; } @@ -1925,4 +1935,5 @@ void vivid_free_controls(struct vivid_dev *dev) v4l2_ctrl_handler_free(&dev->ctrl_hdl_fb); v4l2_ctrl_handler_free(&dev->ctrl_hdl_meta_cap); v4l2_ctrl_handler_free(&dev->ctrl_hdl_meta_out); + v4l2_ctrl_handler_free(&dev->ctrl_hdl_touch_cap); } diff --git a/drivers/media/platform/vivid/vivid-kthread-touch.c b/drivers/media/platform/vivid/vivid-kthread-touch.c new file mode 100644 index 000000000000..674507b5ccb5 --- /dev/null +++ b/drivers/media/platform/vivid/vivid-kthread-touch.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vivid-kthread-touch.c - touch capture thread support functions. + * + */ + +#include <linux/freezer.h> +#include "vivid-core.h" +#include "vivid-kthread-touch.h" +#include "vivid-touch-cap.h" + +static noinline_for_stack void vivid_thread_tch_cap_tick(struct vivid_dev *dev, + int dropped_bufs) +{ + struct vivid_buffer *tch_cap_buf = NULL; + + spin_lock(&dev->slock); + if (!list_empty(&dev->touch_cap_active)) { + tch_cap_buf = list_entry(dev->touch_cap_active.next, + struct vivid_buffer, list); + list_del(&tch_cap_buf->list); + } + + spin_unlock(&dev->slock); + + if (tch_cap_buf) { + v4l2_ctrl_request_setup(tch_cap_buf->vb.vb2_buf.req_obj.req, + &dev->ctrl_hdl_touch_cap); + + vivid_fillbuff_tch(dev, tch_cap_buf); + v4l2_ctrl_request_complete(tch_cap_buf->vb.vb2_buf.req_obj.req, + &dev->ctrl_hdl_touch_cap); + vb2_buffer_done(&tch_cap_buf->vb.vb2_buf, dev->dqbuf_error ? + VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE); + dprintk(dev, 2, "touch_cap buffer %d done\n", + tch_cap_buf->vb.vb2_buf.index); + + tch_cap_buf->vb.vb2_buf.timestamp = ktime_get_ns() + dev->time_wrap_offset; + } + dev->dqbuf_error = false; +} + +static int vivid_thread_touch_cap(void *data) +{ + struct vivid_dev *dev = data; + u64 numerators_since_start; + u64 buffers_since_start; + u64 next_jiffies_since_start; + unsigned long jiffies_since_start; + unsigned long cur_jiffies; + unsigned int wait_jiffies; + unsigned int numerator; + unsigned int denominator; + int dropped_bufs; + + dprintk(dev, 1, "Touch Capture Thread Start\n"); + + set_freezable(); + + /* Resets frame counters */ + dev->touch_cap_seq_offset = 0; + dev->touch_cap_seq_count = 0; + dev->touch_cap_seq_resync = false; + dev->jiffies_touch_cap = jiffies; + + for (;;) { + try_to_freeze(); + if (kthread_should_stop()) + break; + + if (!mutex_trylock(&dev->mutex)) { + schedule_timeout_uninterruptible(1); + continue; + } + cur_jiffies = jiffies; + if (dev->touch_cap_seq_resync) { + dev->jiffies_touch_cap = cur_jiffies; + dev->touch_cap_seq_offset = dev->touch_cap_seq_count + 1; + dev->touch_cap_seq_count = 0; + dev->cap_seq_resync = false; + } + denominator = dev->timeperframe_tch_cap.denominator; + numerator = dev->timeperframe_tch_cap.numerator; + + /* Calculate the number of jiffies since we started streaming */ + jiffies_since_start = cur_jiffies - dev->jiffies_touch_cap; + /* Get the number of buffers streamed since the start */ + buffers_since_start = (u64)jiffies_since_start * denominator + + (HZ * numerator) / 2; + do_div(buffers_since_start, HZ * numerator); + + /* + * After more than 0xf0000000 (rounded down to a multiple of + * 'jiffies-per-day' to ease jiffies_to_msecs calculation) + * jiffies have passed since we started streaming reset the + * counters and keep track of the sequence offset. + */ + if (jiffies_since_start > JIFFIES_RESYNC) { + dev->jiffies_touch_cap = cur_jiffies; + dev->cap_seq_offset = buffers_since_start; + buffers_since_start = 0; + } + dropped_bufs = buffers_since_start + dev->touch_cap_seq_offset - dev->touch_cap_seq_count; + dev->touch_cap_seq_count = buffers_since_start + dev->touch_cap_seq_offset; + + vivid_thread_tch_cap_tick(dev, dropped_bufs); + + /* + * Calculate the number of 'numerators' streamed + * since we started, including the current buffer. + */ + numerators_since_start = ++buffers_since_start * numerator; + + /* And the number of jiffies since we started */ + jiffies_since_start = jiffies - dev->jiffies_touch_cap; + + mutex_unlock(&dev->mutex); + + /* + * Calculate when that next buffer is supposed to start + * in jiffies since we started streaming. + */ + next_jiffies_since_start = numerators_since_start * HZ + + denominator / 2; + do_div(next_jiffies_since_start, denominator); + /* If it is in the past, then just schedule asap */ + if (next_jiffies_since_start < jiffies_since_start) + next_jiffies_since_start = jiffies_since_start; + + wait_jiffies = next_jiffies_since_start - jiffies_since_start; + schedule_timeout_interruptible(wait_jiffies ? wait_jiffies : 1); + } + dprintk(dev, 1, "Touch Capture Thread End\n"); + return 0; +} + +int vivid_start_generating_touch_cap(struct vivid_dev *dev) +{ + if (dev->kthread_touch_cap) { + dev->touch_cap_streaming = true; + return 0; + } + + dev->kthread_touch_cap = kthread_run(vivid_thread_touch_cap, dev, + "%s-tch-cap", dev->v4l2_dev.name); + + if (IS_ERR(dev->kthread_touch_cap)) { + int err = PTR_ERR(dev->kthread_touch_cap); + + dev->kthread_touch_cap = NULL; + v4l2_err(&dev->v4l2_dev, "kernel_thread() failed\n"); + return err; + } + dev->touch_cap_streaming = true; + dprintk(dev, 1, "returning from %s\n", __func__); + return 0; +} + +void vivid_stop_generating_touch_cap(struct vivid_dev *dev) +{ + if (!dev->kthread_touch_cap) + return; + + dev->touch_cap_streaming = false; + + while (!list_empty(&dev->touch_cap_active)) { + struct vivid_buffer *buf; + + buf = list_entry(dev->touch_cap_active.next, + struct vivid_buffer, list); + list_del(&buf->list); + v4l2_ctrl_request_complete(buf->vb.vb2_buf.req_obj.req, + &dev->ctrl_hdl_touch_cap); + vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR); + dprintk(dev, 2, "touch_cap buffer %d done\n", + buf->vb.vb2_buf.index); + } + + kthread_stop(dev->kthread_touch_cap); + dev->kthread_touch_cap = NULL; +} diff --git a/drivers/media/platform/vivid/vivid-kthread-touch.h b/drivers/media/platform/vivid/vivid-kthread-touch.h new file mode 100644 index 000000000000..ecf79b46209e --- /dev/null +++ b/drivers/media/platform/vivid/vivid-kthread-touch.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * vivid-kthread-cap.h - video/vbi capture thread support functions. + * + */ + +#ifndef _VIVID_KTHREAD_CAP_H_ +#define _VIVID_KTHREAD_CAP_H_ + +int vivid_start_generating_touch_cap(struct vivid_dev *dev); +void vivid_stop_generating_touch_cap(struct vivid_dev *dev); + +#endif diff --git a/drivers/media/platform/vivid/vivid-touch-cap.c b/drivers/media/platform/vivid/vivid-touch-cap.c new file mode 100644 index 000000000000..ebb00b128030 --- /dev/null +++ b/drivers/media/platform/vivid/vivid-touch-cap.c @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vivid-touch-cap.c - touch support functions. + */ + +#include "vivid-core.h" +#include "vivid-kthread-touch.h" +#include "vivid-vid-common.h" +#include "vivid-touch-cap.h" + +static int touch_cap_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, + unsigned int *nplanes, unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct vivid_dev *dev = vb2_get_drv_priv(vq); + struct v4l2_pix_format *f = &dev->tch_format; + unsigned int size = f->sizeimage; + + if (*nplanes) { + if (sizes[0] < size) + return -EINVAL; + } else { + sizes[0] = size; + } + + if (vq->num_buffers + *nbuffers < 2) + *nbuffers = 2 - vq->num_buffers; + + *nplanes = 1; + return 0; +} + +static int touch_cap_buf_prepare(struct vb2_buffer *vb) +{ + struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue); + struct v4l2_pix_format *f = &dev->tch_format; + unsigned int size = f->sizeimage; + + if (dev->buf_prepare_error) { + /* + * Error injection: test what happens if buf_prepare() returns + * an error. + */ + dev->buf_prepare_error = false; + return -EINVAL; + } + if (vb2_plane_size(vb, 0) < size) { + dprintk(dev, 1, "%s data will not fit into plane (%lu < %u)\n", + __func__, vb2_plane_size(vb, 0), size); + return -EINVAL; + } + vb2_set_plane_payload(vb, 0, size); + + return 0; +} + +static void touch_cap_buf_queue(struct vb2_buffer *vb) +{ + struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); + struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue); + struct vivid_buffer *buf = container_of(vbuf, struct vivid_buffer, vb); + + vbuf->field = V4L2_FIELD_NONE; + spin_lock(&dev->slock); + list_add_tail(&buf->list, &dev->touch_cap_active); + spin_unlock(&dev->slock); +} + +static int touch_cap_start_streaming(struct vb2_queue *vq, unsigned int count) +{ + struct vivid_dev *dev = vb2_get_drv_priv(vq); + int err; + + dev->touch_cap_seq_count = 0; + if (dev->start_streaming_error) { + dev->start_streaming_error = false; + err = -EINVAL; + } else { + err = vivid_start_generating_touch_cap(dev); + } + if (err) { + struct vivid_buffer *buf, *tmp; + + list_for_each_entry_safe(buf, tmp, + &dev->touch_cap_active, list) { + list_del(&buf->list); + vb2_buffer_done(&buf->vb.vb2_buf, + VB2_BUF_STATE_QUEUED); + } + } + return err; +} + +/* abort streaming and wait for last buffer */ +static void touch_cap_stop_streaming(struct vb2_queue *vq) +{ + struct vivid_dev *dev = vb2_get_drv_priv(vq); + + vivid_stop_generating_touch_cap(dev); +} + +static void touch_cap_buf_request_complete(struct vb2_buffer *vb) +{ + struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue); + + v4l2_ctrl_request_complete(vb->req_obj.req, &dev->ctrl_hdl_touch_cap); +} + +const struct vb2_ops vivid_touch_cap_qops = { + .queue_setup = touch_cap_queue_setup, + .buf_prepare = touch_cap_buf_prepare, + .buf_queue = touch_cap_buf_queue, + .start_streaming = touch_cap_start_streaming, + .stop_streaming = touch_cap_stop_streaming, + .buf_request_complete = touch_cap_buf_request_complete, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, +}; + +int vivid_enum_fmt_tch(struct file *file, void *priv, struct v4l2_fmtdesc *f) +{ + if (f->index) + return -EINVAL; + + f->pixelformat = V4L2_TCH_FMT_DELTA_TD16; + return 0; +} + +int vivid_g_fmt_tch(struct file *file, void *priv, struct v4l2_format *f) +{ + struct vivid_dev *dev = video_drvdata(file); + + if (dev->multiplanar) + return -ENOTTY; + f->fmt.pix = dev->tch_format; + return 0; +} + +int vivid_g_fmt_tch_mplane(struct file *file, void *priv, struct v4l2_format *f) +{ + struct vivid_dev *dev = video_drvdata(file); + struct v4l2_format sp_fmt; + + if (!dev->multiplanar) + return -ENOTTY; + sp_fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + sp_fmt.fmt.pix = dev->tch_format; + fmt_sp2mp(&sp_fmt, f); + return 0; +} + +int vivid_g_parm_tch(struct file *file, void *priv, + struct v4l2_streamparm *parm) +{ + struct vivid_dev *dev = video_drvdata(file); + + if (parm->type != (dev->multiplanar ? + V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE : + V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -EINVAL; + + parm->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; + parm->parm.capture.timeperframe = dev->timeperframe_tch_cap; + parm->parm.capture.readbuffers = 1; + return 0; +} + +int vivid_enum_input_tch(struct file *file, void *priv, struct v4l2_input *inp) +{ + if (inp->index) + return -EINVAL; + + inp->type = V4L2_INPUT_TYPE_TOUCH; + strscpy(inp->name, "Vivid Touch", sizeof(inp->name)); + inp->capabilities = 0; + return 0; +} + +int vivid_g_input_tch(struct file *file, void *priv, unsigned int *i) +{ + *i = 0; + return 0; +} + +int vivid_set_touch(struct vivid_dev *dev, unsigned int i) +{ + struct v4l2_pix_format *f = &dev->tch_format; + + if (i) + return -EINVAL; + + f->pixelformat = V4L2_TCH_FMT_DELTA_TD16; + f->width = VIVID_TCH_WIDTH; + f->height = VIVID_TCH_HEIGHT; + f->field = V4L2_FIELD_NONE; + f->colorspace = V4L2_COLORSPACE_RAW; + f->bytesperline = f->width * sizeof(s16); + f->sizeimage = f->width * f->height * sizeof(s16); + return 0; +} + +int vivid_s_input_tch(struct file *file, void *priv, unsigned int i) +{ + return vivid_set_touch(video_drvdata(file), i); +} + +static void vivid_fill_buff_noise(__s16 *tch_buf, int size) +{ + int i; + + /* Fill 10% of the values within range -3 and 3, zero the others */ + for (i = 0; i < size; i++) { + unsigned int rand = get_random_int(); + + if (rand % 10) + tch_buf[i] = 0; + else + tch_buf[i] = (rand / 10) % 7 - 3; + } +} + +static inline int get_random_pressure(void) +{ + return get_random_int() % VIVID_PRESSURE_LIMIT; +} + +static void vivid_tch_buf_set(struct v4l2_pix_format *f, + __s16 *tch_buf, + int index) +{ + unsigned int x = index % f->width; + unsigned int y = index / f->width; + unsigned int offset = VIVID_MIN_PRESSURE; + + tch_buf[index] = offset + get_random_pressure(); + offset /= 2; + if (x) + tch_buf[index - 1] = offset + get_random_pressure(); + if (x < f->width - 1) + tch_buf[index + 1] = offset + get_random_pressure(); + if (y) + tch_buf[index - f->width] = offset + get_random_pressure(); + if (y < f->height - 1) + tch_buf[index + f->width] = offset + get_random_pressure(); + offset /= 2; + if (x && y) + tch_buf[index - 1 - f->width] = offset + get_random_pressure(); + if (x < f->width - 1 && y) + tch_buf[index + 1 - f->width] = offset + get_random_pressure(); + if (x && y < f->height - 1) + tch_buf[index - 1 + f->width] = offset + get_random_pressure(); + if (x < f->width - 1 && y < f->height - 1) + tch_buf[index + 1 + f->width] = offset + get_random_pressure(); +} + +void vivid_fillbuff_tch(struct vivid_dev *dev, struct vivid_buffer *buf) +{ + struct v4l2_pix_format *f = &dev->tch_format; + int size = f->width * f->height; + int x, y, xstart, ystart, offset_x, offset_y; + unsigned int test_pattern, test_pat_idx, rand; + + __s16 *tch_buf = vb2_plane_vaddr(&buf->vb.vb2_buf, 0); + + buf->vb.sequence = dev->touch_cap_seq_count; + test_pattern = (buf->vb.sequence / TCH_SEQ_COUNT) % TEST_CASE_MAX; + test_pat_idx = buf->vb.sequence % TCH_SEQ_COUNT; + + vivid_fill_buff_noise(tch_buf, size); + + if (test_pat_idx >= TCH_PATTERN_COUNT) + return; + + if (test_pat_idx == 0) + dev->tch_pat_random = get_random_int(); + rand = dev->tch_pat_random; + + switch (test_pattern) { + case SINGLE_TAP: + if (test_pat_idx == 2) + vivid_tch_buf_set(f, tch_buf, rand % size); + break; + case DOUBLE_TAP: + if (test_pat_idx == 2 || test_pat_idx == 4) + vivid_tch_buf_set(f, tch_buf, rand % size); + break; + case TRIPLE_TAP: + if (test_pat_idx == 2 || test_pat_idx == 4 || test_pat_idx == 6) + vivid_tch_buf_set(f, tch_buf, rand % size); + break; + case MOVE_LEFT_TO_RIGHT: + vivid_tch_buf_set(f, tch_buf, + (rand % f->height) * f->width + + test_pat_idx * + (f->width / TCH_PATTERN_COUNT)); + break; + case ZOOM_IN: + x = f->width / 2; + y = f->height / 2; + offset_x = ((TCH_PATTERN_COUNT - 1 - test_pat_idx) * x) / + TCH_PATTERN_COUNT; + offset_y = ((TCH_PATTERN_COUNT - 1 - test_pat_idx) * y) / + TCH_PATTERN_COUNT; + vivid_tch_buf_set(f, tch_buf, + (x - offset_x) + f->width * (y - offset_y)); + vivid_tch_buf_set(f, tch_buf, + (x + offset_x) + f->width * (y + offset_y)); + break; + case ZOOM_OUT: + x = f->width / 2; + y = f->height / 2; + offset_x = (test_pat_idx * x) / TCH_PATTERN_COUNT; + offset_y = (test_pat_idx * y) / TCH_PATTERN_COUNT; + vivid_tch_buf_set(f, tch_buf, + (x - offset_x) + f->width * (y - offset_y)); + vivid_tch_buf_set(f, tch_buf, + (x + offset_x) + f->width * (y + offset_y)); + break; + case PALM_PRESS: + for (x = 0; x < f->width; x++) + for (y = f->height / 2; y < f->height; y++) + tch_buf[x + f->width * y] = VIVID_MIN_PRESSURE + + get_random_pressure(); + break; + case MULTIPLE_PRESS: + /* 16 pressure points */ + for (y = 0; y < 4; y++) { + for (x = 0; x < 4; x++) { + ystart = (y * f->height) / 4 + f->height / 8; + xstart = (x * f->width) / 4 + f->width / 8; + vivid_tch_buf_set(f, tch_buf, + ystart * f->width + xstart); + } + } + break; + } +#ifdef __BIG_ENDIAN__ + for (x = 0; x < size; x++) + tch_buf[x] = (__force s16)__cpu_to_le16((u16)tch_buf[x]); +#endif +} diff --git a/drivers/media/platform/vivid/vivid-touch-cap.h b/drivers/media/platform/vivid/vivid-touch-cap.h new file mode 100644 index 000000000000..07e514046ae8 --- /dev/null +++ b/drivers/media/platform/vivid/vivid-touch-cap.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * vivid-touch-cap.h - touch support functions. + */ +#ifndef _VIVID_TOUCH_CAP_H_ +#define _VIVID_TOUCH_CAP_H_ + +#define VIVID_TCH_HEIGHT 12 +#define VIVID_TCH_WIDTH 21 +#define VIVID_MIN_PRESSURE 180 +#define VIVID_PRESSURE_LIMIT 40 +#define TCH_SEQ_COUNT 16 +#define TCH_PATTERN_COUNT 12 + +enum vivid_tch_test { + SINGLE_TAP, + DOUBLE_TAP, + TRIPLE_TAP, + MOVE_LEFT_TO_RIGHT, + ZOOM_IN, + ZOOM_OUT, + PALM_PRESS, + MULTIPLE_PRESS, + TEST_CASE_MAX +}; + +extern const struct vb2_ops vivid_touch_cap_qops; + +int vivid_enum_fmt_tch(struct file *file, void *priv, struct v4l2_fmtdesc *f); +int vivid_g_fmt_tch(struct file *file, void *priv, struct v4l2_format *f); +int vivid_g_fmt_tch_mplane(struct file *file, void *priv, struct v4l2_format *f); +int vivid_enum_input_tch(struct file *file, void *priv, struct v4l2_input *inp); +int vivid_g_input_tch(struct file *file, void *priv, unsigned int *i); +int vivid_s_input_tch(struct file *file, void *priv, unsigned int i); +void vivid_fillbuff_tch(struct vivid_dev *dev, struct vivid_buffer *buf); +int vivid_set_touch(struct vivid_dev *dev, unsigned int i); +int vivid_g_parm_tch(struct file *file, void *priv, + struct v4l2_streamparm *parm); +#endif diff --git a/drivers/media/platform/vivid/vivid-vid-common.c b/drivers/media/platform/vivid/vivid-vid-common.c index 8665dfd25eb4..76b0be670ebb 100644 --- a/drivers/media/platform/vivid/vivid-vid-common.c +++ b/drivers/media/platform/vivid/vivid-vid-common.c @@ -813,7 +813,7 @@ void fmt_sp2mp(const struct v4l2_format *sp_fmt, struct v4l2_format *mp_fmt) memset(mp->reserved, 0, sizeof(mp->reserved)); mp_fmt->type = is_out ? V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE : - V4L2_CAP_VIDEO_CAPTURE_MPLANE; + V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; mp->width = pix->width; mp->height = pix->height; mp->pixelformat = pix->pixelformat; diff --git a/drivers/media/rc/iguanair.c b/drivers/media/rc/iguanair.c index 872d6441e512..a7deca1fefb7 100644 --- a/drivers/media/rc/iguanair.c +++ b/drivers/media/rc/iguanair.c @@ -413,7 +413,7 @@ static int iguanair_probe(struct usb_interface *intf, int ret, pipein, pipeout; struct usb_host_interface *idesc; - idesc = intf->altsetting; + idesc = intf->cur_altsetting; if (idesc->desc.bNumEndpoints < 2) return -ENODEV; diff --git a/drivers/media/rc/ir-hix5hd2.c b/drivers/media/rc/ir-hix5hd2.c index 32ccefeff57d..d80cfa455c73 100644 --- a/drivers/media/rc/ir-hix5hd2.c +++ b/drivers/media/rc/ir-hix5hd2.c @@ -37,10 +37,13 @@ #define INT_CLR_RCV BIT(16) #define INT_CLR_RCVTIMEOUT (BIT(16) | BIT(17)) -#define IR_CLK 0x48 #define IR_CLK_ENABLE BIT(4) #define IR_CLK_RESET BIT(5) +/* IR_ENABLE register bits */ +#define IR_ENABLE_EN BIT(0) +#define IR_ENABLE_EN_EXTRA BIT(8) + #define IR_CFG_WIDTH_MASK 0xffff #define IR_CFG_WIDTH_SHIFT 16 #define IR_CFG_FORMAT_MASK 0x3 @@ -58,6 +61,23 @@ #define IR_HIX5HD2_NAME "hix5hd2-ir" +/* Need to set extra bit for enabling IR */ +#define HIX5HD2_FLAG_EXTRA_ENABLE BIT(0) + +struct hix5hd2_soc_data { + u32 clk_reg; + u32 flags; +}; + +static const struct hix5hd2_soc_data hix5hd2_data = { + .clk_reg = 0x48, +}; + +static const struct hix5hd2_soc_data hi3796cv300_data = { + .clk_reg = 0x60, + .flags = HIX5HD2_FLAG_EXTRA_ENABLE, +}; + struct hix5hd2_ir_priv { int irq; void __iomem *base; @@ -66,15 +86,17 @@ struct hix5hd2_ir_priv { struct regmap *regmap; struct clk *clock; unsigned long rate; + const struct hix5hd2_soc_data *socdata; }; -static int hix5hd2_ir_enable(struct hix5hd2_ir_priv *dev, bool on) +static int hix5hd2_ir_clk_enable(struct hix5hd2_ir_priv *dev, bool on) { + u32 clk_reg = dev->socdata->clk_reg; u32 val; int ret = 0; if (dev->regmap) { - regmap_read(dev->regmap, IR_CLK, &val); + regmap_read(dev->regmap, clk_reg, &val); if (on) { val &= ~IR_CLK_RESET; val |= IR_CLK_ENABLE; @@ -82,7 +104,7 @@ static int hix5hd2_ir_enable(struct hix5hd2_ir_priv *dev, bool on) val &= ~IR_CLK_ENABLE; val |= IR_CLK_RESET; } - regmap_write(dev->regmap, IR_CLK, val); + regmap_write(dev->regmap, clk_reg, val); } else { if (on) ret = clk_prepare_enable(dev->clock); @@ -92,12 +114,23 @@ static int hix5hd2_ir_enable(struct hix5hd2_ir_priv *dev, bool on) return ret; } +static inline void hix5hd2_ir_enable(struct hix5hd2_ir_priv *priv) +{ + u32 val = IR_ENABLE_EN; + + if (priv->socdata->flags & HIX5HD2_FLAG_EXTRA_ENABLE) + val |= IR_ENABLE_EN_EXTRA; + + writel_relaxed(val, priv->base + IR_ENABLE); +} + static int hix5hd2_ir_config(struct hix5hd2_ir_priv *priv) { int timeout = 10000; u32 val, rate; - writel_relaxed(0x01, priv->base + IR_ENABLE); + hix5hd2_ir_enable(priv); + while (readl_relaxed(priv->base + IR_BUSY)) { if (timeout--) { udelay(1); @@ -128,13 +161,13 @@ static int hix5hd2_ir_open(struct rc_dev *rdev) struct hix5hd2_ir_priv *priv = rdev->priv; int ret; - ret = hix5hd2_ir_enable(priv, true); + ret = hix5hd2_ir_clk_enable(priv, true); if (ret) return ret; ret = hix5hd2_ir_config(priv); if (ret) { - hix5hd2_ir_enable(priv, false); + hix5hd2_ir_clk_enable(priv, false); return ret; } return 0; @@ -144,7 +177,7 @@ static void hix5hd2_ir_close(struct rc_dev *rdev) { struct hix5hd2_ir_priv *priv = rdev->priv; - hix5hd2_ir_enable(priv, false); + hix5hd2_ir_clk_enable(priv, false); } static irqreturn_t hix5hd2_ir_rx_interrupt(int irq, void *data) @@ -205,6 +238,13 @@ static irqreturn_t hix5hd2_ir_rx_interrupt(int irq, void *data) return IRQ_HANDLED; } +static const struct of_device_id hix5hd2_ir_table[] = { + { .compatible = "hisilicon,hix5hd2-ir", &hix5hd2_data, }, + { .compatible = "hisilicon,hi3796cv300-ir", &hi3796cv300_data, }, + {}, +}; +MODULE_DEVICE_TABLE(of, hix5hd2_ir_table); + static int hix5hd2_ir_probe(struct platform_device *pdev) { struct rc_dev *rdev; @@ -212,6 +252,7 @@ static int hix5hd2_ir_probe(struct platform_device *pdev) struct resource *res; struct hix5hd2_ir_priv *priv; struct device_node *node = pdev->dev.of_node; + const struct of_device_id *of_id; const char *map_name; int ret; @@ -219,6 +260,13 @@ static int hix5hd2_ir_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + of_id = of_match_device(hix5hd2_ir_table, dev); + if (!of_id) { + dev_err(dev, "Unable to initialize IR data\n"); + return -ENODEV; + } + priv->socdata = of_id->data; + priv->regmap = syscon_regmap_lookup_by_phandle(node, "hisilicon,power-syscon"); if (IS_ERR(priv->regmap)) { @@ -309,7 +357,7 @@ static int hix5hd2_ir_suspend(struct device *dev) struct hix5hd2_ir_priv *priv = dev_get_drvdata(dev); clk_disable_unprepare(priv->clock); - hix5hd2_ir_enable(priv, false); + hix5hd2_ir_clk_enable(priv, false); return 0; } @@ -319,17 +367,18 @@ static int hix5hd2_ir_resume(struct device *dev) struct hix5hd2_ir_priv *priv = dev_get_drvdata(dev); int ret; - ret = hix5hd2_ir_enable(priv, true); + ret = hix5hd2_ir_clk_enable(priv, true); if (ret) return ret; ret = clk_prepare_enable(priv->clock); if (ret) { - hix5hd2_ir_enable(priv, false); + hix5hd2_ir_clk_enable(priv, false); return ret; } - writel_relaxed(0x01, priv->base + IR_ENABLE); + hix5hd2_ir_enable(priv); + writel_relaxed(0x00, priv->base + IR_INTM); writel_relaxed(0xff, priv->base + IR_INTC); writel_relaxed(0x01, priv->base + IR_START); @@ -341,12 +390,6 @@ static int hix5hd2_ir_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(hix5hd2_ir_pm_ops, hix5hd2_ir_suspend, hix5hd2_ir_resume); -static const struct of_device_id hix5hd2_ir_table[] = { - { .compatible = "hisilicon,hix5hd2-ir", }, - {}, -}; -MODULE_DEVICE_TABLE(of, hix5hd2_ir_table); - static struct platform_driver hix5hd2_ir_driver = { .driver = { .name = IR_HIX5HD2_NAME, diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c index 7741151606ef..6f80c251f641 100644 --- a/drivers/media/rc/rc-main.c +++ b/drivers/media/rc/rc-main.c @@ -1891,23 +1891,28 @@ int rc_register_device(struct rc_dev *dev) dev->registered = true; - if (dev->driver_type != RC_DRIVER_IR_RAW_TX) { - rc = rc_setup_rx_device(dev); - if (rc) - goto out_dev; - } - - /* Ensure that the lirc kfifo is setup before we start the thread */ + /* + * once the the input device is registered in rc_setup_rx_device, + * userspace can open the input device and rc_open() will be called + * as a result. This results in driver code being allowed to submit + * keycodes with rc_keydown, so lirc must be registered first. + */ if (dev->allowed_protocols != RC_PROTO_BIT_CEC) { rc = ir_lirc_register(dev); if (rc < 0) - goto out_rx; + goto out_dev; + } + + if (dev->driver_type != RC_DRIVER_IR_RAW_TX) { + rc = rc_setup_rx_device(dev); + if (rc) + goto out_lirc; } if (dev->driver_type == RC_DRIVER_IR_RAW) { rc = ir_raw_event_register(dev); if (rc < 0) - goto out_lirc; + goto out_rx; } dev_dbg(&dev->dev, "Registered rc%u (driver: %s)\n", dev->minor, @@ -1915,11 +1920,11 @@ int rc_register_device(struct rc_dev *dev) return 0; +out_rx: + rc_free_rx_device(dev); out_lirc: if (dev->allowed_protocols != RC_PROTO_BIT_CEC) ir_lirc_unregister(dev); -out_rx: - rc_free_rx_device(dev); out_dev: device_del(&dev->dev); out_rx_free: diff --git a/drivers/media/rc/serial_ir.c b/drivers/media/rc/serial_ir.c index 7652e982173f..d77507ba0fb5 100644 --- a/drivers/media/rc/serial_ir.c +++ b/drivers/media/rc/serial_ir.c @@ -353,7 +353,7 @@ static irqreturn_t serial_ir_irq_handler(int i, void *blah) dcd = (status & hardware[type].signal_pin) ? 1 : 0; if (dcd == last_dcd) { - dev_err(&serial_ir.pdev->dev, + dev_dbg(&serial_ir.pdev->dev, "ignoring spike: %d %d %lldns %lldns\n", dcd, sense, ktime_to_ns(kt), ktime_to_ns(serial_ir.lastkt)); diff --git a/drivers/media/usb/cpia2/cpia2_v4l.c b/drivers/media/usb/cpia2/cpia2_v4l.c index 626264a56517..9d3d05125d7b 100644 --- a/drivers/media/usb/cpia2/cpia2_v4l.c +++ b/drivers/media/usb/cpia2/cpia2_v4l.c @@ -800,7 +800,7 @@ static int cpia2_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) break; case FRAME_READY: buf->bytesused = cam->buffers[buf->index].length; - buf->timestamp = ns_to_timeval(cam->buffers[buf->index].ts); + v4l2_buffer_set_timestamp(buf, cam->buffers[buf->index].ts); buf->sequence = cam->buffers[buf->index].seq; buf->flags = V4L2_BUF_FLAG_DONE; break; @@ -907,7 +907,7 @@ static int cpia2_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) buf->flags = V4L2_BUF_FLAG_MAPPED | V4L2_BUF_FLAG_DONE | V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; buf->field = V4L2_FIELD_NONE; - buf->timestamp = ns_to_timeval(cam->buffers[buf->index].ts); + v4l2_buffer_set_timestamp(buf, cam->buffers[buf->index].ts); buf->sequence = cam->buffers[buf->index].seq; buf->m.offset = cam->buffers[buf->index].data - cam->frame_buffer; buf->length = cam->frame_size; diff --git a/drivers/media/usb/cx231xx/cx231xx-audio.c b/drivers/media/usb/cx231xx/cx231xx-audio.c index fd6e2df3d1b7..de42db6f6ad1 100644 --- a/drivers/media/usb/cx231xx/cx231xx-audio.c +++ b/drivers/media/usb/cx231xx/cx231xx-audio.c @@ -13,7 +13,6 @@ #include <linux/spinlock.h> #include <linux/soundcard.h> #include <linux/slab.h> -#include <linux/vmalloc.h> #include <linux/module.h> #include <sound/core.h> #include <sound/pcm.h> @@ -372,28 +371,6 @@ static int cx231xx_init_audio_bulk(struct cx231xx *dev) return errCode; } -static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs, - size_t size) -{ - struct snd_pcm_runtime *runtime = subs->runtime; - struct cx231xx *dev = snd_pcm_substream_chip(subs); - - dev_dbg(dev->dev, "Allocating vbuffer\n"); - if (runtime->dma_area) { - if (runtime->dma_bytes > size) - return 0; - - vfree(runtime->dma_area); - } - runtime->dma_area = vmalloc(size); - if (!runtime->dma_area) - return -ENOMEM; - - runtime->dma_bytes = size; - - return 0; -} - static const struct snd_pcm_hardware snd_cx231xx_hw_capture = { .info = SNDRV_PCM_INFO_BLOCK_TRANSFER | SNDRV_PCM_INFO_MMAP | @@ -484,11 +461,6 @@ static int snd_cx231xx_pcm_close(struct snd_pcm_substream *substream) } dev->adev.users--; - if (substream->runtime->dma_area) { - dev_dbg(dev->dev, "freeing\n"); - vfree(substream->runtime->dma_area); - substream->runtime->dma_area = NULL; - } mutex_unlock(&dev->lock); if (dev->adev.users == 0 && dev->adev.shutdown == 1) { @@ -504,44 +476,6 @@ static int snd_cx231xx_pcm_close(struct snd_pcm_substream *substream) return 0; } -static int snd_cx231xx_hw_capture_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *hw_params) -{ - struct cx231xx *dev = snd_pcm_substream_chip(substream); - int ret; - - dev_dbg(dev->dev, "Setting capture parameters\n"); - - ret = snd_pcm_alloc_vmalloc_buffer(substream, - params_buffer_bytes(hw_params)); -#if 0 - /* TODO: set up cx231xx audio chip to deliver the correct audio format, - current default is 48000hz multiplexed => 96000hz mono - which shouldn't matter since analogue TV only supports mono */ - unsigned int channels, rate, format; - - format = params_format(hw_params); - rate = params_rate(hw_params); - channels = params_channels(hw_params); -#endif - - return ret; -} - -static int snd_cx231xx_hw_capture_free(struct snd_pcm_substream *substream) -{ - struct cx231xx *dev = snd_pcm_substream_chip(substream); - - dev_dbg(dev->dev, "Stop capture, if needed\n"); - - if (atomic_read(&dev->stream_started) > 0) { - atomic_set(&dev->stream_started, 0); - schedule_work(&dev->wq_trigger); - } - - return 0; -} - static int snd_cx231xx_prepare(struct snd_pcm_substream *substream) { struct cx231xx *dev = snd_pcm_substream_chip(substream); @@ -614,24 +548,12 @@ static snd_pcm_uframes_t snd_cx231xx_capture_pointer(struct snd_pcm_substream return hwptr_done; } -static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs, - unsigned long offset) -{ - void *pageptr = subs->runtime->dma_area + offset; - - return vmalloc_to_page(pageptr); -} - static const struct snd_pcm_ops snd_cx231xx_pcm_capture = { .open = snd_cx231xx_capture_open, .close = snd_cx231xx_pcm_close, - .ioctl = snd_pcm_lib_ioctl, - .hw_params = snd_cx231xx_hw_capture_params, - .hw_free = snd_cx231xx_hw_capture_free, .prepare = snd_cx231xx_prepare, .trigger = snd_cx231xx_capture_trigger, .pointer = snd_cx231xx_capture_pointer, - .page = snd_pcm_get_vmalloc_page, }; static int cx231xx_audio_init(struct cx231xx *dev) @@ -666,6 +588,7 @@ static int cx231xx_audio_init(struct cx231xx *dev) snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &snd_cx231xx_pcm_capture); + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, NULL, 0, 0); pcm->info_flags = 0; pcm->private_data = dev; strscpy(pcm->name, "Conexant cx231xx Capture", sizeof(pcm->name)); diff --git a/drivers/media/usb/cx231xx/cx231xx-i2c.c b/drivers/media/usb/cx231xx/cx231xx-i2c.c index f33b6a077d57..c6659253c6fb 100644 --- a/drivers/media/usb/cx231xx/cx231xx-i2c.c +++ b/drivers/media/usb/cx231xx/cx231xx-i2c.c @@ -515,7 +515,8 @@ int cx231xx_i2c_register(struct cx231xx_i2c *bus) { struct cx231xx *dev = bus->dev; - BUG_ON(!dev->cx231xx_send_usb_command); + if (!dev->cx231xx_send_usb_command) + return -EINVAL; bus->i2c_adap = cx231xx_adap_template; bus->i2c_adap.dev.parent = dev->dev; diff --git a/drivers/media/usb/dvb-usb-v2/af9035.c b/drivers/media/usb/dvb-usb-v2/af9035.c index 792667ee5ebc..b1f69c11c839 100644 --- a/drivers/media/usb/dvb-usb-v2/af9035.c +++ b/drivers/media/usb/dvb-usb-v2/af9035.c @@ -208,8 +208,8 @@ static int af9035_add_i2c_dev(struct dvb_usb_device *d, const char *type, request_module("%s", board_info.type); /* register I2C device */ - client = i2c_new_device(adapter, &board_info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(adapter, &board_info); + if (!i2c_client_has_driver(client)) { ret = -ENODEV; goto err; } @@ -1621,9 +1621,10 @@ static int it930x_tuner_attach(struct dvb_usb_adapter *adap) si2157_config.fe = adap->fe[0]; /* - * HACK: The Logilink VG0022A has a bug: when the si2157 - * firmware that came with the device is replaced by a new - * one, the I2C transfers to the tuner will return just 0xff. + * HACK: The Logilink VG0022A and TerraTec TC2 Stick have + * a bug: when the si2157 firmware that came with the device + * is replaced by a new one, the I2C transfers to the tuner + * will return just 0xff. * * Probably, the vendor firmware has some patch specifically * designed for this device. So, we can't replace by the @@ -1633,8 +1634,10 @@ static int it930x_tuner_attach(struct dvb_usb_adapter *adap) * while we don't have that, the next best solution is to just * keep the original firmware at the device. */ - if (le16_to_cpu(d->udev->descriptor.idVendor) == USB_VID_DEXATEK && - le16_to_cpu(d->udev->descriptor.idProduct) == 0x0100) + if ((le16_to_cpu(d->udev->descriptor.idVendor) == USB_VID_DEXATEK && + le16_to_cpu(d->udev->descriptor.idProduct) == 0x0100) || + (le16_to_cpu(d->udev->descriptor.idVendor) == USB_VID_TERRATEC && + le16_to_cpu(d->udev->descriptor.idProduct) == USB_PID_TERRATEC_CINERGY_TC2_STICK)) si2157_config.dont_load_firmware = true; si2157_config.if_port = it930x_addresses_table[state->it930x_addresses].tuner_if_port; @@ -2150,6 +2153,8 @@ static const struct usb_device_id af9035_id_table[] = { &it930x_props, "AVerMedia TD310 DVB-T2", NULL) }, { DVB_USB_DEVICE(USB_VID_DEXATEK, 0x0100, &it930x_props, "Logilink VG0022A", NULL) }, + { DVB_USB_DEVICE(USB_VID_TERRATEC, USB_PID_TERRATEC_CINERGY_TC2_STICK, + &it930x_props, "TerraTec Cinergy TC2 Stick", NULL) }, { } }; MODULE_DEVICE_TABLE(usb, af9035_id_table); diff --git a/drivers/media/usb/dvb-usb-v2/anysee.c b/drivers/media/usb/dvb-usb-v2/anysee.c index fb6d99dea31a..0514e87405b6 100644 --- a/drivers/media/usb/dvb-usb-v2/anysee.c +++ b/drivers/media/usb/dvb-usb-v2/anysee.c @@ -649,8 +649,8 @@ static int anysee_add_i2c_dev(struct dvb_usb_device *d, const char *type, request_module("%s", board_info.type); /* register I2C device */ - client = i2c_new_device(adapter, &board_info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(adapter, &board_info); + if (!i2c_client_has_driver(client)) { ret = -ENODEV; goto err; } diff --git a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c index 5016ede7b35f..c6881a1b3232 100644 --- a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c +++ b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c @@ -697,8 +697,8 @@ static int rtl2831u_frontend_attach(struct dvb_usb_adapter *adap) board_info.addr = 0x10; board_info.platform_data = pdata; request_module("%s", board_info.type); - client = i2c_new_device(&d->i2c_adap, &board_info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(&d->i2c_adap, &board_info); + if (!i2c_client_has_driver(client)) { ret = -ENODEV; goto err; } @@ -918,8 +918,8 @@ static int rtl2832u_frontend_attach(struct dvb_usb_adapter *adap) board_info.addr = 0x10; board_info.platform_data = pdata; request_module("%s", board_info.type); - client = i2c_new_device(&d->i2c_adap, &board_info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(&d->i2c_adap, &board_info); + if (!i2c_client_has_driver(client)) { ret = -ENODEV; goto err; } @@ -960,8 +960,8 @@ static int rtl2832u_frontend_attach(struct dvb_usb_adapter *adap) info.addr = 0x18; info.platform_data = &mn88472_config; request_module(info.type); - client = i2c_new_device(&d->i2c_adap, &info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(&d->i2c_adap, &info); + if (!i2c_client_has_driver(client)) { dev->slave_demod = SLAVE_DEMOD_NONE; goto err_slave_demod_failed; } @@ -982,8 +982,8 @@ static int rtl2832u_frontend_attach(struct dvb_usb_adapter *adap) info.addr = 0x18; info.platform_data = &mn88473_config; request_module(info.type); - client = i2c_new_device(&d->i2c_adap, &info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(&d->i2c_adap, &info); + if (!i2c_client_has_driver(client)) { dev->slave_demod = SLAVE_DEMOD_NONE; goto err_slave_demod_failed; } @@ -1025,8 +1025,8 @@ static int rtl2832u_frontend_attach(struct dvb_usb_adapter *adap) info.addr = 0x64; info.platform_data = &si2168_config; request_module(info.type); - client = i2c_new_device(&d->i2c_adap, &info); - if (client == NULL || client->dev.driver == NULL) { + client = i2c_new_client_device(&d->i2c_adap, &info); + if (!i2c_client_has_driver(client)) { dev->slave_demod = SLAVE_DEMOD_NONE; goto err_slave_demod_failed; } @@ -1217,8 +1217,9 @@ static int rtl2832u_tuner_attach(struct dvb_usb_adapter *adap) info.platform_data = &e4000_config; request_module(info.type); - client = i2c_new_device(dev->demod_i2c_adapter, &info); - if (client == NULL || client->dev.driver == NULL) + client = i2c_new_client_device(dev->demod_i2c_adapter, + &info); + if (!i2c_client_has_driver(client)) break; if (!try_module_get(client->dev.driver->owner)) { @@ -1240,9 +1241,9 @@ static int rtl2832u_tuner_attach(struct dvb_usb_adapter *adap) board_info.addr = 0x56; board_info.platform_data = &fc2580_pdata; request_module("fc2580"); - client = i2c_new_device(dev->demod_i2c_adapter, - &board_info); - if (client == NULL || client->dev.driver == NULL) + client = i2c_new_client_device(dev->demod_i2c_adapter, + &board_info); + if (!i2c_client_has_driver(client)) break; if (!try_module_get(client->dev.driver->owner)) { i2c_unregister_device(client); @@ -1271,8 +1272,9 @@ static int rtl2832u_tuner_attach(struct dvb_usb_adapter *adap) board_info.addr = 0x60; board_info.platform_data = &tua9001_pdata; request_module("tua9001"); - client = i2c_new_device(dev->demod_i2c_adapter, &board_info); - if (client == NULL || client->dev.driver == NULL) + client = i2c_new_client_device(dev->demod_i2c_adapter, + &board_info); + if (!i2c_client_has_driver(client)) break; if (!try_module_get(client->dev.driver->owner)) { i2c_unregister_device(client); @@ -1316,8 +1318,8 @@ static int rtl2832u_tuner_attach(struct dvb_usb_adapter *adap) info.addr = 0x60; info.platform_data = &si2157_config; request_module(info.type); - client = i2c_new_device(&d->i2c_adap, &info); - if (client == NULL || client->dev.driver == NULL) + client = i2c_new_client_device(&d->i2c_adap, &info); + if (!i2c_client_has_driver(client)) break; if (!try_module_get(client->dev.driver->owner)) { @@ -1955,6 +1957,8 @@ static const struct usb_device_id rtl28xxu_id_table[] = { &rtl28xxu_props, "Sveon STV27", NULL) }, { DVB_USB_DEVICE(USB_VID_KWORLD_2, USB_PID_TURBOX_DTT_2000, &rtl28xxu_props, "TURBO-X Pure TV Tuner DTT-2000", NULL) }, + { DVB_USB_DEVICE(USB_VID_GTEK, USB_PID_PROLECTRIX_DV107669, + &rtl28xxu_props, "PROlectrix DV107669", NULL) }, /* RTL2832P devices: */ { DVB_USB_DEVICE(USB_VID_HANFTEK, 0x0131, diff --git a/drivers/media/usb/dvb-usb-v2/zd1301.c b/drivers/media/usb/dvb-usb-v2/zd1301.c index 63b66b207b64..ba2c1b0d3989 100644 --- a/drivers/media/usb/dvb-usb-v2/zd1301.c +++ b/drivers/media/usb/dvb-usb-v2/zd1301.c @@ -172,8 +172,8 @@ static int zd1301_frontend_attach(struct dvb_usb_adapter *adap) board_info.addr = 0x60; board_info.platform_data = &dev->mt2060_pdata; request_module("%s", "mt2060"); - client = i2c_new_device(adapter, &board_info); - if (!client || !client->dev.driver) { + client = i2c_new_client_device(adapter, &board_info); + if (!i2c_client_has_driver(client)) { ret = -ENODEV; goto err_module_put_demod; } diff --git a/drivers/media/usb/dvb-usb/af9005.c b/drivers/media/usb/dvb-usb/af9005.c index ac93e88d7038..89b4b5d84cdf 100644 --- a/drivers/media/usb/dvb-usb/af9005.c +++ b/drivers/media/usb/dvb-usb/af9005.c @@ -554,7 +554,7 @@ static int af9005_boot_packet(struct usb_device *udev, int type, u8 *reply, u8 *buf, int size) { u16 checksum; - int act_len, i, ret; + int act_len = 0, i, ret; memset(buf, 0, size); buf[0] = (u8) (FW_BULKOUT_SIZE & 0xff); diff --git a/drivers/media/usb/dvb-usb/cxusb.c b/drivers/media/usb/dvb-usb/cxusb.c index fac19ec46089..c421b603be44 100644 --- a/drivers/media/usb/dvb-usb/cxusb.c +++ b/drivers/media/usb/dvb-usb/cxusb.c @@ -54,9 +54,6 @@ MODULE_PARM_DESC(debug, "set debugging level (see cxusb.h)." DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); -#define deb_info(args...) dprintk(dvb_usb_cxusb_debug, CXUSB_DBG_MISC, args) -#define deb_i2c(args...) dprintk(dvb_usb_cxusb_debug, CXUSB_DBG_I2C, args) - enum cxusb_table_index { MEDION_MD95700, DVICO_BLUEBIRD_LG064F_COLD, @@ -125,7 +122,7 @@ static void cxusb_gpio_tuner(struct dvb_usb_device *d, int onoff) cxusb_ctrl_msg(d, CMD_GPIO_WRITE, o, 2, &i, 1); if (i != 0x01) - deb_info("gpio_write failed.\n"); + dev_info(&d->udev->dev, "gpio_write failed.\n"); st->gpio_write_state[GPIO_TUNER] = onoff; st->gpio_write_refresh[GPIO_TUNER] = false; @@ -142,7 +139,7 @@ static int cxusb_bluebird_gpio_rw(struct dvb_usb_device *d, u8 changemask, rc = cxusb_ctrl_msg(d, CMD_BLUEBIRD_GPIO_RW, o, 2, &gpio_state, 1); if (rc < 0 || (gpio_state & changemask) != (newval & changemask)) - deb_info("bluebird_gpio_write failed.\n"); + dev_info(&d->udev->dev, "bluebird_gpio_write failed.\n"); return rc < 0 ? rc : gpio_state; } @@ -174,7 +171,7 @@ static int cxusb_d680_dmb_gpio_tuner(struct dvb_usb_device *d, if (i == 0x01) return 0; - deb_info("gpio_write failed.\n"); + dev_info(&d->udev->dev, "gpio_write failed.\n"); return -EIO; } @@ -248,7 +245,7 @@ static int cxusb_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msg[], break; if (ibuf[0] != 0x08) - deb_i2c("i2c read may have failed\n"); + dev_info(&d->udev->dev, "i2c read may have failed\n"); memcpy(msg[i + 1].buf, &ibuf[1], msg[i + 1].len); @@ -271,7 +268,7 @@ static int cxusb_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msg[], 2 + msg[i].len, &ibuf, 1) < 0) break; if (ibuf != 0x08) - deb_i2c("i2c write may have failed\n"); + dev_info(&d->udev->dev, "i2c write may have failed\n"); } } @@ -299,7 +296,7 @@ static int _cxusb_power_ctrl(struct dvb_usb_device *d, int onoff) { u8 b = 0; - deb_info("setting power %s\n", onoff ? "ON" : "OFF"); + dev_info(&d->udev->dev, "setting power %s\n", onoff ? "ON" : "OFF"); if (onoff) return cxusb_ctrl_msg(d, CMD_POWER_ON, &b, 1, NULL, 0); @@ -318,7 +315,7 @@ static int cxusb_power_ctrl(struct dvb_usb_device *d, int onoff) mutex_lock(&cxdev->open_lock); if (cxdev->open_type == CXUSB_OPEN_ANALOG) { - deb_info("preventing DVB core from setting power OFF while we are in analog mode\n"); + dev_info(&d->udev->dev, "preventing DVB core from setting power OFF while we are in analog mode\n"); ret = -EBUSY; goto ret_unlock; } @@ -754,16 +751,16 @@ static int dvico_bluebird_xc2028_callback(void *ptr, int component, switch (command) { case XC2028_TUNER_RESET: - deb_info("%s: XC2028_TUNER_RESET %d\n", __func__, arg); + dev_info(&d->udev->dev, "XC2028_TUNER_RESET %d\n", arg); cxusb_bluebird_gpio_pulse(d, 0x01, 1); break; case XC2028_RESET_CLK: - deb_info("%s: XC2028_RESET_CLK %d\n", __func__, arg); + dev_info(&d->udev->dev, "XC2028_RESET_CLK %d\n", arg); break; case XC2028_I2C_FLUSH: break; default: - deb_info("%s: unknown command %d, arg %d\n", __func__, + dev_info(&d->udev->dev, "unknown command %d, arg %d\n", command, arg); return -EINVAL; } @@ -1444,7 +1441,7 @@ int cxusb_medion_get(struct dvb_usb_device *dvbdev, if (cxdev->open_ctr == 0) { if (cxdev->open_type != open_type) { - deb_info("will acquire and switch to %s\n", + dev_info(&dvbdev->udev->dev, "will acquire and switch to %s\n", open_type == CXUSB_OPEN_ANALOG ? "analog" : "digital"); @@ -1476,7 +1473,7 @@ int cxusb_medion_get(struct dvb_usb_device *dvbdev, cxdev->open_type = open_type; } else { - deb_info("reacquired idle %s\n", + dev_info(&dvbdev->udev->dev, "reacquired idle %s\n", open_type == CXUSB_OPEN_ANALOG ? "analog" : "digital"); } @@ -1484,8 +1481,8 @@ int cxusb_medion_get(struct dvb_usb_device *dvbdev, cxdev->open_ctr = 1; } else if (cxdev->open_type == open_type) { cxdev->open_ctr++; - deb_info("acquired %s\n", open_type == CXUSB_OPEN_ANALOG ? - "analog" : "digital"); + dev_info(&dvbdev->udev->dev, "acquired %s\n", + open_type == CXUSB_OPEN_ANALOG ? "analog" : "digital"); } else { ret = -EBUSY; } @@ -1511,7 +1508,7 @@ void cxusb_medion_put(struct dvb_usb_device *dvbdev) if (!WARN_ON(cxdev->open_ctr < 1)) { cxdev->open_ctr--; - deb_info("release %s\n", + dev_info(&dvbdev->udev->dev, "release %s\n", cxdev->open_type == CXUSB_OPEN_ANALOG ? "analog" : "digital"); } diff --git a/drivers/media/usb/dvb-usb/dib0700_devices.c b/drivers/media/usb/dvb-usb/dib0700_devices.c index ab7a100ec84f..4ef3fa98d20f 100644 --- a/drivers/media/usb/dvb-usb/dib0700_devices.c +++ b/drivers/media/usb/dvb-usb/dib0700_devices.c @@ -3772,8 +3772,8 @@ static int xbox_one_attach(struct dvb_usb_adapter *adap) info.addr = 0x18; info.platform_data = &mn88472_config; request_module(info.type); - client_demod = i2c_new_device(&d->i2c_adap, &info); - if (client_demod == NULL || client_demod->dev.driver == NULL) + client_demod = i2c_new_client_device(&d->i2c_adap, &info); + if (!i2c_client_has_driver(client_demod)) goto fail_demod_device; if (!try_module_get(client_demod->dev.driver->owner)) goto fail_demod_module; @@ -3800,8 +3800,8 @@ static int xbox_one_attach(struct dvb_usb_adapter *adap) info.platform_data = &tda18250_config; request_module(info.type); - client_tuner = i2c_new_device(&adap->dev->i2c_adap, &info); - if (client_tuner == NULL || client_tuner->dev.driver == NULL) + client_tuner = i2c_new_client_device(&adap->dev->i2c_adap, &info); + if (!i2c_client_has_driver(client_tuner)) goto fail_tuner_device; if (!try_module_get(client_tuner->dev.driver->owner)) goto fail_tuner_module; diff --git a/drivers/media/usb/dvb-usb/digitv.c b/drivers/media/usb/dvb-usb/digitv.c index dd5bb230cec1..99a39339d45d 100644 --- a/drivers/media/usb/dvb-usb/digitv.c +++ b/drivers/media/usb/dvb-usb/digitv.c @@ -230,18 +230,22 @@ static struct rc_map_table rc_map_digitv_table[] = { static int digitv_rc_query(struct dvb_usb_device *d, u32 *event, int *state) { - int i; + int ret, i; u8 key[5]; u8 b[4] = { 0 }; *event = 0; *state = REMOTE_NO_KEY_PRESSED; - digitv_ctrl_msg(d,USB_READ_REMOTE,0,NULL,0,&key[1],4); + ret = digitv_ctrl_msg(d, USB_READ_REMOTE, 0, NULL, 0, &key[1], 4); + if (ret) + return ret; /* Tell the device we've read the remote. Not sure how necessary this is, but the Nebula SDK does it. */ - digitv_ctrl_msg(d,USB_WRITE_REMOTE,0,b,4,NULL,0); + ret = digitv_ctrl_msg(d, USB_WRITE_REMOTE, 0, b, 4, NULL, 0); + if (ret) + return ret; /* if something is inside the buffer, simulate key press */ if (key[1] != 0) diff --git a/drivers/media/usb/dvb-usb/dvb-usb-urb.c b/drivers/media/usb/dvb-usb/dvb-usb-urb.c index c1b4e94a37f8..2aabf90d8697 100644 --- a/drivers/media/usb/dvb-usb/dvb-usb-urb.c +++ b/drivers/media/usb/dvb-usb/dvb-usb-urb.c @@ -12,7 +12,7 @@ int dvb_usb_generic_rw(struct dvb_usb_device *d, u8 *wbuf, u16 wlen, u8 *rbuf, u16 rlen, int delay_ms) { - int actlen,ret = -ENOMEM; + int actlen = 0, ret = -ENOMEM; if (!d || wbuf == NULL || wlen == 0) return -EINVAL; diff --git a/drivers/media/usb/dvb-usb/dw2102.c b/drivers/media/usb/dvb-usb/dw2102.c index b960abd00d48..8b584507dd59 100644 --- a/drivers/media/usb/dvb-usb/dw2102.c +++ b/drivers/media/usb/dvb-usb/dw2102.c @@ -1590,8 +1590,8 @@ static int tt_s2_4600_frontend_attach(struct dvb_usb_adapter *adap) board_info.addr = 0x68; board_info.platform_data = &m88ds3103_pdata; request_module("m88ds3103"); - client = i2c_new_device(&d->i2c_adap, &board_info); - if (client == NULL || client->dev.driver == NULL) + client = i2c_new_client_device(&d->i2c_adap, &board_info); + if (!i2c_client_has_driver(client)) return -ENODEV; if (!try_module_get(client->dev.driver->owner)) { i2c_unregister_device(client); @@ -1609,9 +1609,9 @@ static int tt_s2_4600_frontend_attach(struct dvb_usb_adapter *adap) board_info.addr = 0x60; board_info.platform_data = &ts2020_config; request_module("ts2020"); - client = i2c_new_device(i2c_adapter, &board_info); + client = i2c_new_client_device(i2c_adapter, &board_info); - if (client == NULL || client->dev.driver == NULL) { + if (!i2c_client_has_driver(client)) { dvb_frontend_detach(adap->fe_adap[0].fe); return -ENODEV; } diff --git a/drivers/media/usb/dvb-usb/vp7045.c b/drivers/media/usb/dvb-usb/vp7045.c index 80c1cf05384b..2baf57216d19 100644 --- a/drivers/media/usb/dvb-usb/vp7045.c +++ b/drivers/media/usb/dvb-usb/vp7045.c @@ -96,10 +96,14 @@ static int vp7045_power_ctrl(struct dvb_usb_device *d, int onoff) static int vp7045_rc_query(struct dvb_usb_device *d) { + int ret; u8 key; - vp7045_usb_op(d,RC_VAL_READ,NULL,0,&key,1,20); - deb_rc("remote query key: %x %d\n",key,key); + ret = vp7045_usb_op(d, RC_VAL_READ, NULL, 0, &key, 1, 20); + if (ret) + return ret; + + deb_rc("remote query key: %x\n", key); if (key != 0x44) { /* @@ -115,15 +119,18 @@ static int vp7045_rc_query(struct dvb_usb_device *d) static int vp7045_read_eeprom(struct dvb_usb_device *d,u8 *buf, int len, int offset) { - int i = 0; - u8 v,br[2]; + int i, ret; + u8 v, br[2]; for (i=0; i < len; i++) { v = offset + i; - vp7045_usb_op(d,GET_EE_VALUE,&v,1,br,2,5); + ret = vp7045_usb_op(d, GET_EE_VALUE, &v, 1, br, 2, 5); + if (ret) + return ret; + buf[i] = br[1]; } - deb_info("VP7045 EEPROM read (offs: %d, len: %d) : ",offset, i); - debug_dump(buf,i,deb_info); + deb_info("VP7045 EEPROM read (offs: %d, len: %d) : ", offset, i); + debug_dump(buf, i, deb_info); return 0; } diff --git a/drivers/media/usb/em28xx/em28xx-audio.c b/drivers/media/usb/em28xx/em28xx-audio.c index 79dfbb25714b..6833b5bfe293 100644 --- a/drivers/media/usb/em28xx/em28xx-audio.c +++ b/drivers/media/usb/em28xx/em28xx-audio.c @@ -30,7 +30,6 @@ #include <linux/spinlock.h> #include <linux/soundcard.h> #include <linux/slab.h> -#include <linux/vmalloc.h> #include <linux/module.h> #include <sound/core.h> #include <sound/pcm.h> @@ -192,28 +191,6 @@ static int em28xx_init_audio_isoc(struct em28xx *dev) return 0; } -static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs, - size_t size) -{ - struct em28xx *dev = snd_pcm_substream_chip(subs); - struct snd_pcm_runtime *runtime = subs->runtime; - - dprintk("Allocating vbuffer\n"); - if (runtime->dma_area) { - if (runtime->dma_bytes > size) - return 0; - - vfree(runtime->dma_area); - } - runtime->dma_area = vmalloc(size); - if (!runtime->dma_area) - return -ENOMEM; - - runtime->dma_bytes = size; - - return 0; -} - static const struct snd_pcm_hardware snd_em28xx_hw_capture = { .info = SNDRV_PCM_INFO_BLOCK_TRANSFER | SNDRV_PCM_INFO_MMAP | @@ -341,63 +318,12 @@ static int snd_em28xx_pcm_close(struct snd_pcm_substream *substream) } em28xx_audio_analog_set(dev); - if (substream->runtime->dma_area) { - dprintk("freeing\n"); - vfree(substream->runtime->dma_area); - substream->runtime->dma_area = NULL; - } mutex_unlock(&dev->lock); kref_put(&dev->ref, em28xx_free_device); return 0; } -static int snd_em28xx_hw_capture_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *hw_params) -{ - int ret; - struct em28xx *dev = snd_pcm_substream_chip(substream); - - if (dev->disconnected) - return -ENODEV; - - dprintk("Setting capture parameters\n"); - - ret = snd_pcm_alloc_vmalloc_buffer(substream, - params_buffer_bytes(hw_params)); - if (ret < 0) - return ret; -#if 0 - /* - * TODO: set up em28xx audio chip to deliver the correct audio format, - * current default is 48000hz multiplexed => 96000hz mono - * which shouldn't matter since analogue TV only supports mono - */ - unsigned int channels, rate, format; - - format = params_format(hw_params); - rate = params_rate(hw_params); - channels = params_channels(hw_params); -#endif - - return 0; -} - -static int snd_em28xx_hw_capture_free(struct snd_pcm_substream *substream) -{ - struct em28xx *dev = snd_pcm_substream_chip(substream); - struct em28xx_audio *adev = &dev->adev; - - dprintk("Stop capture, if needed\n"); - - if (atomic_read(&adev->stream_started) > 0) { - atomic_set(&adev->stream_started, 0); - schedule_work(&adev->wq_trigger); - } - - return 0; -} - static int snd_em28xx_prepare(struct snd_pcm_substream *substream) { struct em28xx *dev = snd_pcm_substream_chip(substream); @@ -471,14 +397,6 @@ static snd_pcm_uframes_t snd_em28xx_capture_pointer(struct snd_pcm_substream return hwptr_done; } -static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs, - unsigned long offset) -{ - void *pageptr = subs->runtime->dma_area + offset; - - return vmalloc_to_page(pageptr); -} - /* * AC97 volume control support */ @@ -708,13 +626,9 @@ static int em28xx_cvol_new(struct snd_card *card, struct em28xx *dev, static const struct snd_pcm_ops snd_em28xx_pcm_capture = { .open = snd_em28xx_capture_open, .close = snd_em28xx_pcm_close, - .ioctl = snd_pcm_lib_ioctl, - .hw_params = snd_em28xx_hw_capture_params, - .hw_free = snd_em28xx_hw_capture_free, .prepare = snd_em28xx_prepare, .trigger = snd_em28xx_capture_trigger, .pointer = snd_em28xx_capture_pointer, - .page = snd_pcm_get_vmalloc_page, }; static void em28xx_audio_free_urb(struct em28xx *dev) @@ -936,6 +850,7 @@ static int em28xx_audio_init(struct em28xx *dev) goto card_free; snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &snd_em28xx_pcm_capture); + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, NULL, 0, 0); pcm->info_flags = 0; pcm->private_data = dev; strscpy(pcm->name, "Empia 28xx Capture", sizeof(pcm->name)); diff --git a/drivers/media/usb/go7007/s2250-board.c b/drivers/media/usb/go7007/s2250-board.c index 49e75a1a1f3f..b9e45124673b 100644 --- a/drivers/media/usb/go7007/s2250-board.c +++ b/drivers/media/usb/go7007/s2250-board.c @@ -607,6 +607,7 @@ static int s2250_remove(struct i2c_client *client) { struct s2250 *state = to_state(i2c_get_clientdata(client)); + i2c_unregister_device(state->audio); v4l2_device_unregister_subdev(&state->sd); v4l2_ctrl_handler_free(&state->hdl); kfree(state); diff --git a/drivers/media/usb/go7007/snd-go7007.c b/drivers/media/usb/go7007/snd-go7007.c index b05fa227ffb2..2ce85ab38db5 100644 --- a/drivers/media/usb/go7007/snd-go7007.c +++ b/drivers/media/usb/go7007/snd-go7007.c @@ -9,7 +9,6 @@ #include <linux/spinlock.h> #include <linux/delay.h> #include <linux/sched.h> -#include <linux/vmalloc.h> #include <linux/time.h> #include <linux/mm.h> #include <linux/i2c.h> @@ -100,16 +99,7 @@ static int go7007_snd_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params) { struct go7007 *go = snd_pcm_substream_chip(substream); - unsigned int bytes; - - bytes = params_buffer_bytes(hw_params); - if (substream->runtime->dma_bytes > 0) - vfree(substream->runtime->dma_area); - substream->runtime->dma_bytes = 0; - substream->runtime->dma_area = vmalloc(bytes); - if (substream->runtime->dma_area == NULL) - return -ENOMEM; - substream->runtime->dma_bytes = bytes; + go->audio_deliver = parse_audio_stream_data; return 0; } @@ -119,9 +109,6 @@ static int go7007_snd_hw_free(struct snd_pcm_substream *substream) struct go7007 *go = snd_pcm_substream_chip(substream); go->audio_deliver = NULL; - if (substream->runtime->dma_bytes > 0) - vfree(substream->runtime->dma_area); - substream->runtime->dma_bytes = 0; return 0; } @@ -185,22 +172,14 @@ static snd_pcm_uframes_t go7007_snd_pcm_pointer(struct snd_pcm_substream *substr return gosnd->hw_ptr; } -static struct page *go7007_snd_pcm_page(struct snd_pcm_substream *substream, - unsigned long offset) -{ - return vmalloc_to_page(substream->runtime->dma_area + offset); -} - static const struct snd_pcm_ops go7007_snd_capture_ops = { .open = go7007_snd_capture_open, .close = go7007_snd_capture_close, - .ioctl = snd_pcm_lib_ioctl, .hw_params = go7007_snd_hw_params, .hw_free = go7007_snd_hw_free, .prepare = go7007_snd_pcm_prepare, .trigger = go7007_snd_pcm_trigger, .pointer = go7007_snd_pcm_pointer, - .page = go7007_snd_pcm_page, }; static int go7007_snd_free(struct snd_device *device) @@ -236,22 +215,18 @@ int go7007_snd_init(struct go7007 *go) gosnd->capturing = 0; ret = snd_card_new(go->dev, index[dev], id[dev], THIS_MODULE, 0, &gosnd->card); - if (ret < 0) { - kfree(gosnd); - return ret; - } + if (ret < 0) + goto free_snd; + ret = snd_device_new(gosnd->card, SNDRV_DEV_LOWLEVEL, go, &go7007_snd_device_ops); - if (ret < 0) { - kfree(gosnd); - return ret; - } + if (ret < 0) + goto free_card; + ret = snd_pcm_new(gosnd->card, "go7007", 0, 0, 1, &gosnd->pcm); - if (ret < 0) { - snd_card_free(gosnd->card); - kfree(gosnd); - return ret; - } + if (ret < 0) + goto free_card; + strscpy(gosnd->card->driver, "go7007", sizeof(gosnd->card->driver)); strscpy(gosnd->card->shortname, go->name, sizeof(gosnd->card->shortname)); strscpy(gosnd->card->longname, gosnd->card->shortname, @@ -260,13 +235,12 @@ int go7007_snd_init(struct go7007 *go) gosnd->pcm->private_data = go; snd_pcm_set_ops(gosnd->pcm, SNDRV_PCM_STREAM_CAPTURE, &go7007_snd_capture_ops); + snd_pcm_set_managed_buffer_all(gosnd->pcm, SNDRV_DMA_TYPE_VMALLOC, + NULL, 0, 0); ret = snd_card_register(gosnd->card); - if (ret < 0) { - snd_card_free(gosnd->card); - kfree(gosnd); - return ret; - } + if (ret < 0) + goto free_card; gosnd->substream = NULL; go->snd_context = gosnd; @@ -274,6 +248,12 @@ int go7007_snd_init(struct go7007 *go) ++dev; return 0; + +free_card: + snd_card_free(gosnd->card); +free_snd: + kfree(gosnd); + return ret; } EXPORT_SYMBOL(go7007_snd_init); diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c index 4add2b12d330..c1b307bbe540 100644 --- a/drivers/media/usb/gspca/gspca.c +++ b/drivers/media/usb/gspca/gspca.c @@ -1461,7 +1461,7 @@ int gspca_dev_probe2(struct usb_interface *intf, pr_err("couldn't kzalloc gspca struct\n"); return -ENOMEM; } - gspca_dev->usb_buf = kmalloc(USB_BUF_SZ, GFP_KERNEL); + gspca_dev->usb_buf = kzalloc(USB_BUF_SZ, GFP_KERNEL); if (!gspca_dev->usb_buf) { pr_err("out of memory\n"); ret = -ENOMEM; diff --git a/drivers/media/usb/pulse8-cec/pulse8-cec.c b/drivers/media/usb/pulse8-cec/pulse8-cec.c index 59609556d969..afda438d4e0a 100644 --- a/drivers/media/usb/pulse8-cec/pulse8-cec.c +++ b/drivers/media/usb/pulse8-cec/pulse8-cec.c @@ -49,7 +49,7 @@ static int debug; static int persistent_config; module_param(debug, int, 0644); module_param(persistent_config, int, 0644); -MODULE_PARM_DESC(debug, "debug level (0-1)"); +MODULE_PARM_DESC(debug, "debug level (0-2)"); MODULE_PARM_DESC(persistent_config, "read config from persistent memory (0-1)"); enum pulse8_msgcodes { @@ -100,6 +100,61 @@ enum pulse8_msgcodes { MSGCODE_FRAME_ACK = 0x40, }; +static const char * const pulse8_msgnames[] = { + "NOTHING", + "PING", + "TIMEOUT_ERROR", + "HIGH_ERROR", + "LOW_ERROR", + "FRAME_START", + "FRAME_DATA", + "RECEIVE_FAILED", + "COMMAND_ACCEPTED", + "COMMAND_REJECTED", + "SET_ACK_MASK", + "TRANSMIT", + "TRANSMIT_EOM", + "TRANSMIT_IDLETIME", + "TRANSMIT_ACK_POLARITY", + "TRANSMIT_LINE_TIMEOUT", + "TRANSMIT_SUCCEEDED", + "TRANSMIT_FAILED_LINE", + "TRANSMIT_FAILED_ACK", + "TRANSMIT_FAILED_TIMEOUT_DATA", + "TRANSMIT_FAILED_TIMEOUT_LINE", + "FIRMWARE_VERSION", + "START_BOOTLOADER", + "GET_BUILDDATE", + "SET_CONTROLLED", + "GET_AUTO_ENABLED", + "SET_AUTO_ENABLED", + "GET_DEFAULT_LOGICAL_ADDRESS", + "SET_DEFAULT_LOGICAL_ADDRESS", + "GET_LOGICAL_ADDRESS_MASK", + "SET_LOGICAL_ADDRESS_MASK", + "GET_PHYSICAL_ADDRESS", + "SET_PHYSICAL_ADDRESS", + "GET_DEVICE_TYPE", + "SET_DEVICE_TYPE", + "GET_HDMI_VERSION", + "SET_HDMI_VERSION", + "GET_OSD_NAME", + "SET_OSD_NAME", + "WRITE_EEPROM", + "GET_ADAPTER_TYPE", + "SET_ACTIVE_SOURCE", +}; + +static const char *pulse8_msgname(u8 cmd) +{ + static char unknown_msg[5]; + + if ((cmd & 0x3f) < ARRAY_SIZE(pulse8_msgnames)) + return pulse8_msgnames[cmd & 0x3f]; + snprintf(unknown_msg, sizeof(unknown_msg), "0x%02x", cmd); + return unknown_msg; +} + #define MSGSTART 0xff #define MSGEND 0xfe #define MSGESC 0xfd @@ -109,60 +164,203 @@ enum pulse8_msgcodes { #define PING_PERIOD (15 * HZ) +#define NUM_MSGS 8 + struct pulse8 { struct device *dev; struct serio *serio; struct cec_adapter *adap; unsigned int vers; - struct completion cmd_done; - struct work_struct work; - u8 work_result; + struct delayed_work ping_eeprom_work; - struct cec_msg rx_msg; + + struct work_struct irq_work; + struct cec_msg rx_msg[NUM_MSGS]; + unsigned int rx_msg_cur_idx, rx_msg_num; + /* protect rx_msg_cur_idx and rx_msg_num */ + spinlock_t msg_lock; + u8 new_rx_msg[CEC_MAX_MSG_SIZE]; + u8 new_rx_msg_len; + + struct work_struct tx_work; + u32 tx_done_status; + u32 tx_signal_free_time; + struct cec_msg tx_msg; + bool tx_msg_is_bcast; + + struct completion cmd_done; u8 data[DATA_SIZE]; unsigned int len; u8 buf[DATA_SIZE]; unsigned int idx; bool escape; bool started; - struct mutex config_lock; - struct mutex write_lock; + + /* locks access to the adapter */ + struct mutex lock; bool config_pending; bool restoring_config; bool autonomous; }; -static void pulse8_ping_eeprom_work_handler(struct work_struct *work); +static int pulse8_send(struct serio *serio, const u8 *command, u8 cmd_len) +{ + int err = 0; + + err = serio_write(serio, MSGSTART); + if (err) + return err; + for (; !err && cmd_len; command++, cmd_len--) { + if (*command >= MSGESC) { + err = serio_write(serio, MSGESC); + if (!err) + err = serio_write(serio, *command - MSGOFFSET); + } else { + err = serio_write(serio, *command); + } + } + if (!err) + err = serio_write(serio, MSGEND); + + return err; +} + +static int pulse8_send_and_wait_once(struct pulse8 *pulse8, + const u8 *cmd, u8 cmd_len, + u8 response, u8 size) +{ + int err; + + if (debug > 1) + dev_info(pulse8->dev, "transmit %s: %*ph\n", + pulse8_msgname(cmd[0]), cmd_len, cmd); + init_completion(&pulse8->cmd_done); + + err = pulse8_send(pulse8->serio, cmd, cmd_len); + if (err) + return err; + + if (!wait_for_completion_timeout(&pulse8->cmd_done, HZ)) + return -ETIMEDOUT; + if ((pulse8->data[0] & 0x3f) == MSGCODE_COMMAND_REJECTED && + cmd[0] != MSGCODE_SET_CONTROLLED && + cmd[0] != MSGCODE_SET_AUTO_ENABLED && + cmd[0] != MSGCODE_GET_BUILDDATE) + return -ENOTTY; + if (response && + ((pulse8->data[0] & 0x3f) != response || pulse8->len < size + 1)) { + dev_info(pulse8->dev, "transmit %s failed with %s\n", + pulse8_msgname(cmd[0]), + pulse8_msgname(pulse8->data[0])); + return -EIO; + } + return 0; +} + +static int pulse8_send_and_wait(struct pulse8 *pulse8, + const u8 *cmd, u8 cmd_len, u8 response, u8 size) +{ + u8 cmd_sc[2]; + int err; + + err = pulse8_send_and_wait_once(pulse8, cmd, cmd_len, response, size); + if (err != -ENOTTY) + return err; + + cmd_sc[0] = MSGCODE_SET_CONTROLLED; + cmd_sc[1] = 1; + err = pulse8_send_and_wait_once(pulse8, cmd_sc, 2, + MSGCODE_COMMAND_ACCEPTED, 1); + if (!err) + err = pulse8_send_and_wait_once(pulse8, cmd, cmd_len, + response, size); + return err == -ENOTTY ? -EIO : err; +} + +static void pulse8_tx_work_handler(struct work_struct *work) +{ + struct pulse8 *pulse8 = container_of(work, struct pulse8, tx_work); + struct cec_msg *msg = &pulse8->tx_msg; + unsigned int i; + u8 cmd[2]; + int err; + + if (msg->len == 0) + return; + + mutex_lock(&pulse8->lock); + cmd[0] = MSGCODE_TRANSMIT_IDLETIME; + cmd[1] = pulse8->tx_signal_free_time; + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 1); + cmd[0] = MSGCODE_TRANSMIT_ACK_POLARITY; + cmd[1] = cec_msg_is_broadcast(msg); + pulse8->tx_msg_is_bcast = cec_msg_is_broadcast(msg); + if (!err) + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 1); + cmd[0] = msg->len == 1 ? MSGCODE_TRANSMIT_EOM : MSGCODE_TRANSMIT; + cmd[1] = msg->msg[0]; + if (!err) + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 1); + if (!err && msg->len > 1) { + for (i = 1; !err && i < msg->len; i++) { + cmd[0] = ((i == msg->len - 1)) ? + MSGCODE_TRANSMIT_EOM : MSGCODE_TRANSMIT; + cmd[1] = msg->msg[i]; + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 1); + } + } + if (err && debug) + dev_info(pulse8->dev, "%s(0x%02x) failed with error %d for msg %*ph\n", + pulse8_msgname(cmd[0]), cmd[1], + err, msg->len, msg->msg); + msg->len = 0; + mutex_unlock(&pulse8->lock); + if (err) + cec_transmit_attempt_done(pulse8->adap, CEC_TX_STATUS_ERROR); +} static void pulse8_irq_work_handler(struct work_struct *work) { struct pulse8 *pulse8 = - container_of(work, struct pulse8, work); - u8 result = pulse8->work_result; + container_of(work, struct pulse8, irq_work); + unsigned long flags; + u32 status; - pulse8->work_result = 0; - switch (result & 0x3f) { - case MSGCODE_FRAME_DATA: - cec_received_msg(pulse8->adap, &pulse8->rx_msg); - break; - case MSGCODE_TRANSMIT_SUCCEEDED: - cec_transmit_attempt_done(pulse8->adap, CEC_TX_STATUS_OK); - break; - case MSGCODE_TRANSMIT_FAILED_ACK: - cec_transmit_attempt_done(pulse8->adap, CEC_TX_STATUS_NACK); - break; - case MSGCODE_TRANSMIT_FAILED_LINE: - case MSGCODE_TRANSMIT_FAILED_TIMEOUT_DATA: - case MSGCODE_TRANSMIT_FAILED_TIMEOUT_LINE: - cec_transmit_attempt_done(pulse8->adap, CEC_TX_STATUS_ERROR); - break; + spin_lock_irqsave(&pulse8->msg_lock, flags); + while (pulse8->rx_msg_num) { + spin_unlock_irqrestore(&pulse8->msg_lock, flags); + if (debug) + dev_info(pulse8->dev, "adap received %*ph\n", + pulse8->rx_msg[pulse8->rx_msg_cur_idx].len, + pulse8->rx_msg[pulse8->rx_msg_cur_idx].msg); + cec_received_msg(pulse8->adap, + &pulse8->rx_msg[pulse8->rx_msg_cur_idx]); + spin_lock_irqsave(&pulse8->msg_lock, flags); + if (pulse8->rx_msg_num) + pulse8->rx_msg_num--; + pulse8->rx_msg_cur_idx = + (pulse8->rx_msg_cur_idx + 1) % NUM_MSGS; } + spin_unlock_irqrestore(&pulse8->msg_lock, flags); + + mutex_lock(&pulse8->lock); + status = pulse8->tx_done_status; + pulse8->tx_done_status = 0; + mutex_unlock(&pulse8->lock); + if (status) + cec_transmit_attempt_done(pulse8->adap, status); } static irqreturn_t pulse8_interrupt(struct serio *serio, unsigned char data, unsigned int flags) { struct pulse8 *pulse8 = serio_get_drvdata(serio); + unsigned long irq_flags; + unsigned int idx; if (!pulse8->started && data != MSGSTART) return IRQ_HANDLED; @@ -174,41 +372,80 @@ static irqreturn_t pulse8_interrupt(struct serio *serio, unsigned char data, data += MSGOFFSET; pulse8->escape = false; } else if (data == MSGEND) { - struct cec_msg *msg = &pulse8->rx_msg; u8 msgcode = pulse8->buf[0]; - if (debug) - dev_info(pulse8->dev, "received: %*ph\n", + if (debug > 1) + dev_info(pulse8->dev, "received %s: %*ph\n", + pulse8_msgname(msgcode), pulse8->idx, pulse8->buf); switch (msgcode & 0x3f) { case MSGCODE_FRAME_START: - msg->len = 1; - msg->msg[0] = pulse8->buf[1]; - break; + /* + * Test if we are receiving a new msg when a previous + * message is still pending. + */ + if (!(msgcode & MSGCODE_FRAME_EOM)) { + pulse8->new_rx_msg_len = 1; + pulse8->new_rx_msg[0] = pulse8->buf[1]; + break; + } + /* fall through */ case MSGCODE_FRAME_DATA: - if (msg->len == CEC_MAX_MSG_SIZE) + if (pulse8->new_rx_msg_len < CEC_MAX_MSG_SIZE) + pulse8->new_rx_msg[pulse8->new_rx_msg_len++] = + pulse8->buf[1]; + if (!(msgcode & MSGCODE_FRAME_EOM)) break; - msg->msg[msg->len++] = pulse8->buf[1]; - if (msgcode & MSGCODE_FRAME_EOM) { - WARN_ON(pulse8->work_result); - pulse8->work_result = msgcode; - schedule_work(&pulse8->work); + + spin_lock_irqsave(&pulse8->msg_lock, irq_flags); + idx = (pulse8->rx_msg_cur_idx + pulse8->rx_msg_num) % + NUM_MSGS; + if (pulse8->rx_msg_num == NUM_MSGS) { + dev_warn(pulse8->dev, + "message queue is full, dropping %*ph\n", + pulse8->new_rx_msg_len, + pulse8->new_rx_msg); + spin_unlock_irqrestore(&pulse8->msg_lock, + irq_flags); + pulse8->new_rx_msg_len = 0; break; } + pulse8->rx_msg_num++; + memcpy(pulse8->rx_msg[idx].msg, pulse8->new_rx_msg, + pulse8->new_rx_msg_len); + pulse8->rx_msg[idx].len = pulse8->new_rx_msg_len; + spin_unlock_irqrestore(&pulse8->msg_lock, irq_flags); + schedule_work(&pulse8->irq_work); + pulse8->new_rx_msg_len = 0; break; case MSGCODE_TRANSMIT_SUCCEEDED: - case MSGCODE_TRANSMIT_FAILED_LINE: + WARN_ON(pulse8->tx_done_status); + pulse8->tx_done_status = CEC_TX_STATUS_OK; + schedule_work(&pulse8->irq_work); + break; case MSGCODE_TRANSMIT_FAILED_ACK: + /* + * A NACK for a broadcast message makes no sense, these + * seem to be spurious messages and are skipped. + */ + if (pulse8->tx_msg_is_bcast) + break; + WARN_ON(pulse8->tx_done_status); + pulse8->tx_done_status = CEC_TX_STATUS_NACK; + schedule_work(&pulse8->irq_work); + break; + case MSGCODE_TRANSMIT_FAILED_LINE: case MSGCODE_TRANSMIT_FAILED_TIMEOUT_DATA: case MSGCODE_TRANSMIT_FAILED_TIMEOUT_LINE: - WARN_ON(pulse8->work_result); - pulse8->work_result = msgcode; - schedule_work(&pulse8->work); + WARN_ON(pulse8->tx_done_status); + pulse8->tx_done_status = CEC_TX_STATUS_ERROR; + schedule_work(&pulse8->irq_work); break; case MSGCODE_HIGH_ERROR: case MSGCODE_LOW_ERROR: case MSGCODE_RECEIVE_FAILED: case MSGCODE_TIMEOUT_ERROR: + pulse8->new_rx_msg_len = 0; break; case MSGCODE_COMMAND_ACCEPTED: case MSGCODE_COMMAND_REJECTED: @@ -238,92 +475,183 @@ static irqreturn_t pulse8_interrupt(struct serio *serio, unsigned char data, return IRQ_HANDLED; } -static void pulse8_disconnect(struct serio *serio) +static int pulse8_cec_adap_enable(struct cec_adapter *adap, bool enable) { - struct pulse8 *pulse8 = serio_get_drvdata(serio); + struct pulse8 *pulse8 = cec_get_drvdata(adap); + u8 cmd[16]; + int err; - cec_unregister_adapter(pulse8->adap); - cancel_delayed_work_sync(&pulse8->ping_eeprom_work); - dev_info(&serio->dev, "disconnected\n"); - serio_close(serio); - serio_set_drvdata(serio, NULL); - kfree(pulse8); + mutex_lock(&pulse8->lock); + cmd[0] = MSGCODE_SET_CONTROLLED; + cmd[1] = enable; + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 1); + if (!enable) { + pulse8->rx_msg_num = 0; + pulse8->tx_done_status = 0; + } + mutex_unlock(&pulse8->lock); + return enable ? err : 0; } -static int pulse8_send(struct serio *serio, const u8 *command, u8 cmd_len) +static int pulse8_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr) { + struct pulse8 *pulse8 = cec_get_drvdata(adap); + u16 mask = 0; + u16 pa = adap->phys_addr; + u8 cmd[16]; int err = 0; - err = serio_write(serio, MSGSTART); + mutex_lock(&pulse8->lock); + if (log_addr != CEC_LOG_ADDR_INVALID) + mask = 1 << log_addr; + cmd[0] = MSGCODE_SET_ACK_MASK; + cmd[1] = mask >> 8; + cmd[2] = mask & 0xff; + err = pulse8_send_and_wait(pulse8, cmd, 3, + MSGCODE_COMMAND_ACCEPTED, 0); + if ((err && mask != 0) || pulse8->restoring_config) + goto unlock; + + cmd[0] = MSGCODE_SET_AUTO_ENABLED; + cmd[1] = log_addr == CEC_LOG_ADDR_INVALID ? 0 : 1; + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 0); if (err) - return err; - for (; !err && cmd_len; command++, cmd_len--) { - if (*command >= MSGESC) { - err = serio_write(serio, MSGESC); - if (!err) - err = serio_write(serio, *command - MSGOFFSET); - } else { - err = serio_write(serio, *command); - } - } - if (!err) - err = serio_write(serio, MSGEND); + goto unlock; + pulse8->autonomous = cmd[1]; + if (log_addr == CEC_LOG_ADDR_INVALID) + goto unlock; - return err; -} + cmd[0] = MSGCODE_SET_DEVICE_TYPE; + cmd[1] = adap->log_addrs.primary_device_type[0]; + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 0); + if (err) + goto unlock; -static int pulse8_send_and_wait_once(struct pulse8 *pulse8, - const u8 *cmd, u8 cmd_len, - u8 response, u8 size) -{ - int err; + switch (adap->log_addrs.primary_device_type[0]) { + case CEC_OP_PRIM_DEVTYPE_TV: + mask = CEC_LOG_ADDR_MASK_TV; + break; + case CEC_OP_PRIM_DEVTYPE_RECORD: + mask = CEC_LOG_ADDR_MASK_RECORD; + break; + case CEC_OP_PRIM_DEVTYPE_TUNER: + mask = CEC_LOG_ADDR_MASK_TUNER; + break; + case CEC_OP_PRIM_DEVTYPE_PLAYBACK: + mask = CEC_LOG_ADDR_MASK_PLAYBACK; + break; + case CEC_OP_PRIM_DEVTYPE_AUDIOSYSTEM: + mask = CEC_LOG_ADDR_MASK_AUDIOSYSTEM; + break; + case CEC_OP_PRIM_DEVTYPE_SWITCH: + mask = CEC_LOG_ADDR_MASK_UNREGISTERED; + break; + case CEC_OP_PRIM_DEVTYPE_PROCESSOR: + mask = CEC_LOG_ADDR_MASK_SPECIFIC; + break; + default: + mask = 0; + break; + } + cmd[0] = MSGCODE_SET_LOGICAL_ADDRESS_MASK; + cmd[1] = mask >> 8; + cmd[2] = mask & 0xff; + err = pulse8_send_and_wait(pulse8, cmd, 3, + MSGCODE_COMMAND_ACCEPTED, 0); + if (err) + goto unlock; - /*dev_info(pulse8->dev, "transmit: %*ph\n", cmd_len, cmd);*/ - init_completion(&pulse8->cmd_done); + cmd[0] = MSGCODE_SET_DEFAULT_LOGICAL_ADDRESS; + cmd[1] = log_addr; + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 0); + if (err) + goto unlock; - err = pulse8_send(pulse8->serio, cmd, cmd_len); + cmd[0] = MSGCODE_SET_PHYSICAL_ADDRESS; + cmd[1] = pa >> 8; + cmd[2] = pa & 0xff; + err = pulse8_send_and_wait(pulse8, cmd, 3, + MSGCODE_COMMAND_ACCEPTED, 0); if (err) - return err; + goto unlock; - if (!wait_for_completion_timeout(&pulse8->cmd_done, HZ)) - return -ETIMEDOUT; - if ((pulse8->data[0] & 0x3f) == MSGCODE_COMMAND_REJECTED && - cmd[0] != MSGCODE_SET_CONTROLLED && - cmd[0] != MSGCODE_SET_AUTO_ENABLED && - cmd[0] != MSGCODE_GET_BUILDDATE) - return -ENOTTY; - if (response && - ((pulse8->data[0] & 0x3f) != response || pulse8->len < size + 1)) { - dev_info(pulse8->dev, "transmit: failed %02x\n", - pulse8->data[0] & 0x3f); - return -EIO; + cmd[0] = MSGCODE_SET_HDMI_VERSION; + cmd[1] = adap->log_addrs.cec_version; + err = pulse8_send_and_wait(pulse8, cmd, 2, + MSGCODE_COMMAND_ACCEPTED, 0); + if (err) + goto unlock; + + if (adap->log_addrs.osd_name[0]) { + size_t osd_len = strlen(adap->log_addrs.osd_name); + char *osd_str = cmd + 1; + + cmd[0] = MSGCODE_SET_OSD_NAME; + strscpy(cmd + 1, adap->log_addrs.osd_name, sizeof(cmd) - 1); + if (osd_len < 4) { + memset(osd_str + osd_len, ' ', 4 - osd_len); + osd_len = 4; + osd_str[osd_len] = '\0'; + strscpy(adap->log_addrs.osd_name, osd_str, + sizeof(adap->log_addrs.osd_name)); + } + err = pulse8_send_and_wait(pulse8, cmd, 1 + osd_len, + MSGCODE_COMMAND_ACCEPTED, 0); + if (err) + goto unlock; } + +unlock: + if (pulse8->restoring_config) + pulse8->restoring_config = false; + else + pulse8->config_pending = true; + mutex_unlock(&pulse8->lock); + return log_addr == CEC_LOG_ADDR_INVALID ? 0 : err; +} + +static int pulse8_cec_adap_transmit(struct cec_adapter *adap, u8 attempts, + u32 signal_free_time, struct cec_msg *msg) +{ + struct pulse8 *pulse8 = cec_get_drvdata(adap); + + pulse8->tx_msg = *msg; + if (debug) + dev_info(pulse8->dev, "adap transmit %*ph\n", + msg->len, msg->msg); + pulse8->tx_signal_free_time = signal_free_time; + schedule_work(&pulse8->tx_work); return 0; } -static int pulse8_send_and_wait(struct pulse8 *pulse8, - const u8 *cmd, u8 cmd_len, u8 response, u8 size) +static void pulse8_cec_adap_free(struct cec_adapter *adap) { - u8 cmd_sc[2]; - int err; + struct pulse8 *pulse8 = cec_get_drvdata(adap); - mutex_lock(&pulse8->write_lock); - err = pulse8_send_and_wait_once(pulse8, cmd, cmd_len, response, size); + cancel_delayed_work_sync(&pulse8->ping_eeprom_work); + cancel_work_sync(&pulse8->irq_work); + cancel_work_sync(&pulse8->tx_work); + serio_close(pulse8->serio); + serio_set_drvdata(pulse8->serio, NULL); + kfree(pulse8); +} - if (err == -ENOTTY) { - cmd_sc[0] = MSGCODE_SET_CONTROLLED; - cmd_sc[1] = 1; - err = pulse8_send_and_wait_once(pulse8, cmd_sc, 2, - MSGCODE_COMMAND_ACCEPTED, 1); - if (err) - goto unlock; - err = pulse8_send_and_wait_once(pulse8, cmd, cmd_len, - response, size); - } +static const struct cec_adap_ops pulse8_cec_adap_ops = { + .adap_enable = pulse8_cec_adap_enable, + .adap_log_addr = pulse8_cec_adap_log_addr, + .adap_transmit = pulse8_cec_adap_transmit, + .adap_free = pulse8_cec_adap_free, +}; -unlock: - mutex_unlock(&pulse8->write_lock); - return err == -ENOTTY ? -EIO : err; +static void pulse8_disconnect(struct serio *serio) +{ + struct pulse8 *pulse8 = serio_get_drvdata(serio); + + cec_unregister_adapter(pulse8->adap); } static int pulse8_setup(struct pulse8 *pulse8, struct serio *serio, @@ -460,191 +788,34 @@ static int pulse8_apply_persistent_config(struct pulse8 *pulse8, return 0; } -static int pulse8_cec_adap_enable(struct cec_adapter *adap, bool enable) -{ - struct pulse8 *pulse8 = cec_get_drvdata(adap); - u8 cmd[16]; - int err; - - cmd[0] = MSGCODE_SET_CONTROLLED; - cmd[1] = enable; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 1); - return enable ? err : 0; -} - -static int pulse8_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr) +static void pulse8_ping_eeprom_work_handler(struct work_struct *work) { - struct pulse8 *pulse8 = cec_get_drvdata(adap); - u16 mask = 0; - u16 pa = adap->phys_addr; - u8 cmd[16]; - int err = 0; - - mutex_lock(&pulse8->config_lock); - if (log_addr != CEC_LOG_ADDR_INVALID) - mask = 1 << log_addr; - cmd[0] = MSGCODE_SET_ACK_MASK; - cmd[1] = mask >> 8; - cmd[2] = mask & 0xff; - err = pulse8_send_and_wait(pulse8, cmd, 3, - MSGCODE_COMMAND_ACCEPTED, 0); - if ((err && mask != 0) || pulse8->restoring_config) - goto unlock; - - cmd[0] = MSGCODE_SET_AUTO_ENABLED; - cmd[1] = log_addr == CEC_LOG_ADDR_INVALID ? 0 : 1; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 0); - if (err) - goto unlock; - pulse8->autonomous = cmd[1]; - if (log_addr == CEC_LOG_ADDR_INVALID) - goto unlock; - - cmd[0] = MSGCODE_SET_DEVICE_TYPE; - cmd[1] = adap->log_addrs.primary_device_type[0]; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 0); - if (err) - goto unlock; - - switch (adap->log_addrs.primary_device_type[0]) { - case CEC_OP_PRIM_DEVTYPE_TV: - mask = CEC_LOG_ADDR_MASK_TV; - break; - case CEC_OP_PRIM_DEVTYPE_RECORD: - mask = CEC_LOG_ADDR_MASK_RECORD; - break; - case CEC_OP_PRIM_DEVTYPE_TUNER: - mask = CEC_LOG_ADDR_MASK_TUNER; - break; - case CEC_OP_PRIM_DEVTYPE_PLAYBACK: - mask = CEC_LOG_ADDR_MASK_PLAYBACK; - break; - case CEC_OP_PRIM_DEVTYPE_AUDIOSYSTEM: - mask = CEC_LOG_ADDR_MASK_AUDIOSYSTEM; - break; - case CEC_OP_PRIM_DEVTYPE_SWITCH: - mask = CEC_LOG_ADDR_MASK_UNREGISTERED; - break; - case CEC_OP_PRIM_DEVTYPE_PROCESSOR: - mask = CEC_LOG_ADDR_MASK_SPECIFIC; - break; - default: - mask = 0; - break; - } - cmd[0] = MSGCODE_SET_LOGICAL_ADDRESS_MASK; - cmd[1] = mask >> 8; - cmd[2] = mask & 0xff; - err = pulse8_send_and_wait(pulse8, cmd, 3, - MSGCODE_COMMAND_ACCEPTED, 0); - if (err) - goto unlock; - - cmd[0] = MSGCODE_SET_DEFAULT_LOGICAL_ADDRESS; - cmd[1] = log_addr; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 0); - if (err) - goto unlock; + struct pulse8 *pulse8 = + container_of(work, struct pulse8, ping_eeprom_work.work); + u8 cmd; - cmd[0] = MSGCODE_SET_PHYSICAL_ADDRESS; - cmd[1] = pa >> 8; - cmd[2] = pa & 0xff; - err = pulse8_send_and_wait(pulse8, cmd, 3, - MSGCODE_COMMAND_ACCEPTED, 0); - if (err) - goto unlock; + mutex_lock(&pulse8->lock); + cmd = MSGCODE_PING; + pulse8_send_and_wait(pulse8, &cmd, 1, + MSGCODE_COMMAND_ACCEPTED, 0); - cmd[0] = MSGCODE_SET_HDMI_VERSION; - cmd[1] = adap->log_addrs.cec_version; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 0); - if (err) + if (pulse8->vers < 2) goto unlock; - if (adap->log_addrs.osd_name[0]) { - size_t osd_len = strlen(adap->log_addrs.osd_name); - char *osd_str = cmd + 1; - - cmd[0] = MSGCODE_SET_OSD_NAME; - strscpy(cmd + 1, adap->log_addrs.osd_name, sizeof(cmd) - 1); - if (osd_len < 4) { - memset(osd_str + osd_len, ' ', 4 - osd_len); - osd_len = 4; - osd_str[osd_len] = '\0'; - strscpy(adap->log_addrs.osd_name, osd_str, - sizeof(adap->log_addrs.osd_name)); - } - err = pulse8_send_and_wait(pulse8, cmd, 1 + osd_len, - MSGCODE_COMMAND_ACCEPTED, 0); - if (err) - goto unlock; + if (pulse8->config_pending && persistent_config) { + dev_dbg(pulse8->dev, "writing pending config to EEPROM\n"); + cmd = MSGCODE_WRITE_EEPROM; + if (pulse8_send_and_wait(pulse8, &cmd, 1, + MSGCODE_COMMAND_ACCEPTED, 0)) + dev_info(pulse8->dev, "failed to write pending config to EEPROM\n"); + else + pulse8->config_pending = false; } - unlock: - if (pulse8->restoring_config) - pulse8->restoring_config = false; - else - pulse8->config_pending = true; - mutex_unlock(&pulse8->config_lock); - return log_addr == CEC_LOG_ADDR_INVALID ? 0 : err; -} - -static int pulse8_cec_adap_transmit(struct cec_adapter *adap, u8 attempts, - u32 signal_free_time, struct cec_msg *msg) -{ - struct pulse8 *pulse8 = cec_get_drvdata(adap); - u8 cmd[2]; - unsigned int i; - int err; - - cmd[0] = MSGCODE_TRANSMIT_IDLETIME; - cmd[1] = signal_free_time; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 1); - cmd[0] = MSGCODE_TRANSMIT_ACK_POLARITY; - cmd[1] = cec_msg_is_broadcast(msg); - if (!err) - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 1); - cmd[0] = msg->len == 1 ? MSGCODE_TRANSMIT_EOM : MSGCODE_TRANSMIT; - cmd[1] = msg->msg[0]; - if (!err) - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 1); - if (!err && msg->len > 1) { - cmd[0] = msg->len == 2 ? MSGCODE_TRANSMIT_EOM : - MSGCODE_TRANSMIT; - cmd[1] = msg->msg[1]; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 1); - for (i = 0; !err && i + 2 < msg->len; i++) { - cmd[0] = (i + 2 == msg->len - 1) ? - MSGCODE_TRANSMIT_EOM : MSGCODE_TRANSMIT; - cmd[1] = msg->msg[i + 2]; - err = pulse8_send_and_wait(pulse8, cmd, 2, - MSGCODE_COMMAND_ACCEPTED, 1); - } - } - - return err; -} - -static int pulse8_received(struct cec_adapter *adap, struct cec_msg *msg) -{ - return -ENOMSG; + schedule_delayed_work(&pulse8->ping_eeprom_work, PING_PERIOD); + mutex_unlock(&pulse8->lock); } -static const struct cec_adap_ops pulse8_cec_adap_ops = { - .adap_enable = pulse8_cec_adap_enable, - .adap_log_addr = pulse8_cec_adap_log_addr, - .adap_transmit = pulse8_cec_adap_transmit, - .received = pulse8_received, -}; - static int pulse8_connect(struct serio *serio, struct serio_driver *drv) { u32 caps = CEC_CAP_DEFAULTS | CEC_CAP_PHYS_ADDR | CEC_CAP_MONITOR_ALL; @@ -667,9 +838,10 @@ static int pulse8_connect(struct serio *serio, struct serio_driver *drv) pulse8->dev = &serio->dev; serio_set_drvdata(serio, pulse8); - INIT_WORK(&pulse8->work, pulse8_irq_work_handler); - mutex_init(&pulse8->write_lock); - mutex_init(&pulse8->config_lock); + INIT_WORK(&pulse8->irq_work, pulse8_irq_work_handler); + INIT_WORK(&pulse8->tx_work, pulse8_tx_work_handler); + mutex_init(&pulse8->lock); + spin_lock_init(&pulse8->msg_lock); pulse8->config_pending = false; err = serio_open(serio, drv); @@ -709,33 +881,6 @@ free_device: return err; } -static void pulse8_ping_eeprom_work_handler(struct work_struct *work) -{ - struct pulse8 *pulse8 = - container_of(work, struct pulse8, ping_eeprom_work.work); - u8 cmd; - - schedule_delayed_work(&pulse8->ping_eeprom_work, PING_PERIOD); - cmd = MSGCODE_PING; - pulse8_send_and_wait(pulse8, &cmd, 1, - MSGCODE_COMMAND_ACCEPTED, 0); - - if (pulse8->vers < 2) - return; - - mutex_lock(&pulse8->config_lock); - if (pulse8->config_pending && persistent_config) { - dev_dbg(pulse8->dev, "writing pending config to EEPROM\n"); - cmd = MSGCODE_WRITE_EEPROM; - if (pulse8_send_and_wait(pulse8, &cmd, 1, - MSGCODE_COMMAND_ACCEPTED, 0)) - dev_info(pulse8->dev, "failed to write pending config to EEPROM\n"); - else - pulse8->config_pending = false; - } - mutex_unlock(&pulse8->config_lock); -} - static const struct serio_device_id pulse8_serio_ids[] = { { .type = SERIO_RS232, diff --git a/drivers/media/usb/pvrusb2/pvrusb2-encoder.c b/drivers/media/usb/pvrusb2/pvrusb2-encoder.c index fb3178d909ce..f6005d1296ef 100644 --- a/drivers/media/usb/pvrusb2/pvrusb2-encoder.c +++ b/drivers/media/usb/pvrusb2/pvrusb2-encoder.c @@ -284,8 +284,8 @@ rdData[0]); wrData[0] = 0x0; ret = pvr2_encoder_write_words(hdw,MBOX_BASE,wrData,1); break; - - }; LOCK_GIVE(hdw->ctl_lock); + } + LOCK_GIVE(hdw->ctl_lock); return ret; } diff --git a/drivers/media/usb/stkwebcam/stk-webcam.c b/drivers/media/usb/stkwebcam/stk-webcam.c index 21f90a887485..b22501f76b78 100644 --- a/drivers/media/usb/stkwebcam/stk-webcam.c +++ b/drivers/media/usb/stkwebcam/stk-webcam.c @@ -1125,7 +1125,7 @@ static int stk_vidioc_dqbuf(struct file *filp, sbuf->v4lbuf.flags &= ~V4L2_BUF_FLAG_QUEUED; sbuf->v4lbuf.flags |= V4L2_BUF_FLAG_DONE; sbuf->v4lbuf.sequence = ++dev->sequence; - sbuf->v4lbuf.timestamp = ns_to_timeval(ktime_get_ns()); + v4l2_buffer_set_timestamp(&sbuf->v4lbuf, ktime_get_ns()); *buf = sbuf->v4lbuf; return 0; diff --git a/drivers/media/usb/tm6000/tm6000-alsa.c b/drivers/media/usb/tm6000/tm6000-alsa.c index d6c79c13b332..c26a0ff60a64 100644 --- a/drivers/media/usb/tm6000/tm6000-alsa.c +++ b/drivers/media/usb/tm6000/tm6000-alsa.c @@ -10,7 +10,6 @@ #include <linux/interrupt.h> #include <linux/usb.h> #include <linux/slab.h> -#include <linux/vmalloc.h> #include <linux/delay.h> #include <sound/core.h> @@ -94,40 +93,6 @@ static int _tm6000_stop_audio_dma(struct snd_tm6000_card *chip) return 0; } -static void dsp_buffer_free(struct snd_pcm_substream *substream) -{ - struct snd_tm6000_card *chip = snd_pcm_substream_chip(substream); - - dprintk(2, "Freeing buffer\n"); - - vfree(substream->runtime->dma_area); - substream->runtime->dma_area = NULL; - substream->runtime->dma_bytes = 0; -} - -static int dsp_buffer_alloc(struct snd_pcm_substream *substream, int size) -{ - struct snd_tm6000_card *chip = snd_pcm_substream_chip(substream); - - dprintk(2, "Allocating buffer\n"); - - if (substream->runtime->dma_area) { - if (substream->runtime->dma_bytes > size) - return 0; - - dsp_buffer_free(substream); - } - - substream->runtime->dma_area = vmalloc(size); - if (!substream->runtime->dma_area) - return -ENOMEM; - - substream->runtime->dma_bytes = size; - - return 0; -} - - /**************************************************************************** ALSA PCM Interface ****************************************************************************/ @@ -269,40 +234,6 @@ static int tm6000_fillbuf(struct tm6000_core *core, char *buf, int size) } /* - * hw_params callback - */ -static int snd_tm6000_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *hw_params) -{ - int size, rc; - - size = params_period_bytes(hw_params) * params_periods(hw_params); - - rc = dsp_buffer_alloc(substream, size); - if (rc < 0) - return rc; - - return 0; -} - -/* - * hw free callback - */ -static int snd_tm6000_hw_free(struct snd_pcm_substream *substream) -{ - struct snd_tm6000_card *chip = snd_pcm_substream_chip(substream); - struct tm6000_core *core = chip->core; - - if (atomic_read(&core->stream_started) > 0) { - atomic_set(&core->stream_started, 0); - schedule_work(&core->wq_trigger); - } - - dsp_buffer_free(substream); - return 0; -} - -/* * prepare callback */ static int snd_tm6000_prepare(struct snd_pcm_substream *substream) @@ -369,27 +300,15 @@ static snd_pcm_uframes_t snd_tm6000_pointer(struct snd_pcm_substream *substream) return chip->buf_pos; } -static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs, - unsigned long offset) -{ - void *pageptr = subs->runtime->dma_area + offset; - - return vmalloc_to_page(pageptr); -} - /* * operators */ static const struct snd_pcm_ops snd_tm6000_pcm_ops = { .open = snd_tm6000_pcm_open, .close = snd_tm6000_close, - .ioctl = snd_pcm_lib_ioctl, - .hw_params = snd_tm6000_hw_params, - .hw_free = snd_tm6000_hw_free, .prepare = snd_tm6000_prepare, .trigger = snd_tm6000_card_trigger, .pointer = snd_tm6000_pointer, - .page = snd_pcm_get_vmalloc_page, }; /* @@ -459,6 +378,7 @@ static int tm6000_audio_init(struct tm6000_core *dev) strscpy(pcm->name, "Trident TM5600/60x0", sizeof(pcm->name)); snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &snd_tm6000_pcm_ops); + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, NULL, 0, 0); INIT_WORK(&dev->wq_trigger, audio_trigger); rc = snd_card_register(card); diff --git a/drivers/media/usb/usbtv/usbtv-audio.c b/drivers/media/usb/usbtv/usbtv-audio.c index e746c8ddfc49..b57e94fb1977 100644 --- a/drivers/media/usb/usbtv/usbtv-audio.c +++ b/drivers/media/usb/usbtv/usbtv-audio.c @@ -85,30 +85,6 @@ static int snd_usbtv_pcm_close(struct snd_pcm_substream *substream) return 0; } -static int snd_usbtv_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *hw_params) -{ - int rv; - struct usbtv *chip = snd_pcm_substream_chip(substream); - - rv = snd_pcm_lib_malloc_pages(substream, - params_buffer_bytes(hw_params)); - - if (rv < 0) { - dev_warn(chip->dev, "pcm audio buffer allocation failure %i\n", - rv); - return rv; - } - - return 0; -} - -static int snd_usbtv_hw_free(struct snd_pcm_substream *substream) -{ - snd_pcm_lib_free_pages(substream); - return 0; -} - static int snd_usbtv_prepare(struct snd_pcm_substream *substream) { struct usbtv *chip = snd_pcm_substream_chip(substream); @@ -336,9 +312,6 @@ static snd_pcm_uframes_t snd_usbtv_pointer(struct snd_pcm_substream *substream) static const struct snd_pcm_ops snd_usbtv_pcm_ops = { .open = snd_usbtv_pcm_open, .close = snd_usbtv_pcm_close, - .ioctl = snd_pcm_lib_ioctl, - .hw_params = snd_usbtv_hw_params, - .hw_free = snd_usbtv_hw_free, .prepare = snd_usbtv_prepare, .trigger = snd_usbtv_card_trigger, .pointer = snd_usbtv_pointer, @@ -377,7 +350,7 @@ int usbtv_audio_init(struct usbtv *usbtv) pcm->private_data = usbtv; snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &snd_usbtv_pcm_ops); - snd_pcm_lib_preallocate_pages_for_all(pcm, SNDRV_DMA_TYPE_CONTINUOUS, + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_CONTINUOUS, NULL, USBTV_AUDIO_BUFFER, USBTV_AUDIO_BUFFER); rv = snd_card_register(card); diff --git a/drivers/media/usb/usbvision/usbvision-video.c b/drivers/media/usb/usbvision/usbvision-video.c index 93d36aab824f..5ca2c2f35fe2 100644 --- a/drivers/media/usb/usbvision/usbvision-video.c +++ b/drivers/media/usb/usbvision/usbvision-video.c @@ -696,7 +696,7 @@ static int vidioc_querybuf(struct file *file, vb->length = usbvision->curwidth * usbvision->curheight * usbvision->palette.bytes_per_pixel; - vb->timestamp = ns_to_timeval(usbvision->frame[vb->index].ts); + v4l2_buffer_set_timestamp(vb, usbvision->frame[vb->index].ts); vb->sequence = usbvision->frame[vb->index].sequence; return 0; } @@ -765,7 +765,7 @@ static int vidioc_dqbuf(struct file *file, void *priv, struct v4l2_buffer *vb) V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; vb->index = f->index; vb->sequence = f->sequence; - vb->timestamp = ns_to_timeval(f->ts); + v4l2_buffer_set_timestamp(vb, f->ts); vb->field = V4L2_FIELD_NONE; vb->bytesused = f->scanlength; diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c index 428235ca2635..99883550375e 100644 --- a/drivers/media/usb/uvc/uvc_driver.c +++ b/drivers/media/usb/uvc/uvc_driver.c @@ -497,6 +497,22 @@ static int uvc_parse_format(struct uvc_device *dev, } } + /* Some devices report bpp that doesn't match the format. */ + if (dev->quirks & UVC_QUIRK_FORCE_BPP) { + const struct v4l2_format_info *info = + v4l2_format_info(format->fcc); + + if (info) { + unsigned int div = info->hdiv * info->vdiv; + + n = info->bpp[0] * div; + for (i = 1; i < info->comp_planes; i++) + n += info->bpp[i]; + + format->bpp = DIV_ROUND_UP(8 * n, div); + } + } + if (buffer[2] == UVC_VS_FORMAT_UNCOMPRESSED) { ftype = UVC_VS_FRAME_UNCOMPRESSED; } else { @@ -1493,6 +1509,11 @@ static int uvc_scan_chain_forward(struct uvc_video_chain *chain, break; if (forward == prev) continue; + if (forward->chain.next || forward->chain.prev) { + uvc_trace(UVC_TRACE_DESCR, "Found reference to " + "entity %d already in chain.\n", forward->id); + return -EINVAL; + } switch (UVC_ENTITY_TYPE(forward)) { case UVC_VC_EXTENSION_UNIT: @@ -1574,6 +1595,13 @@ static int uvc_scan_chain_backward(struct uvc_video_chain *chain, return -1; } + if (term->chain.next || term->chain.prev) { + uvc_trace(UVC_TRACE_DESCR, "Found reference to " + "entity %d already in chain.\n", + term->id); + return -EINVAL; + } + if (uvc_trace_param & UVC_TRACE_PROBE) printk(KERN_CONT " %d", term->id); @@ -2862,6 +2890,15 @@ static const struct usb_device_id uvc_ids[] = { .bInterfaceSubClass = 1, .bInterfaceProtocol = 0, .driver_info = (kernel_ulong_t)&uvc_quirk_force_y8 }, + /* GEO Semiconductor GC6500 */ + { .match_flags = USB_DEVICE_ID_MATCH_DEVICE + | USB_DEVICE_ID_MATCH_INT_INFO, + .idVendor = 0x29fe, + .idProduct = 0x4d53, + .bInterfaceClass = USB_CLASS_VIDEO, + .bInterfaceSubClass = 1, + .bInterfaceProtocol = 0, + .driver_info = UVC_INFO_QUIRK(UVC_QUIRK_FORCE_BPP) }, /* Intel RealSense D4M */ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE | USB_DEVICE_ID_MATCH_INT_INFO, diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h index f773dc5d802c..6ab972c643e3 100644 --- a/drivers/media/usb/uvc/uvcvideo.h +++ b/drivers/media/usb/uvc/uvcvideo.h @@ -198,6 +198,7 @@ #define UVC_QUIRK_RESTRICT_FRAME_RATE 0x00000200 #define UVC_QUIRK_RESTORE_CTRLS_ON_INIT 0x00000400 #define UVC_QUIRK_FORCE_Y8 0x00000800 +#define UVC_QUIRK_FORCE_BPP 0x00001000 /* Format flags */ #define UVC_FMT_FLAG_COMPRESSED 0x00000001 diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c index e1eaf1135c7f..a99e82ec9ab6 100644 --- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c +++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c @@ -468,13 +468,43 @@ struct v4l2_plane32 { __u32 reserved[11]; }; +/* + * This is correct for all architectures including i386, but not x32, + * which has different alignment requirements for timestamp + */ struct v4l2_buffer32 { __u32 index; __u32 type; /* enum v4l2_buf_type */ __u32 bytesused; __u32 flags; __u32 field; /* enum v4l2_field */ - struct compat_timeval timestamp; + struct { + compat_s64 tv_sec; + compat_s64 tv_usec; + } timestamp; + struct v4l2_timecode timecode; + __u32 sequence; + + /* memory location */ + __u32 memory; /* enum v4l2_memory */ + union { + __u32 offset; + compat_long_t userptr; + compat_caddr_t planes; + __s32 fd; + } m; + __u32 length; + __u32 reserved2; + __s32 request_fd; +}; + +struct v4l2_buffer32_time32 { + __u32 index; + __u32 type; /* enum v4l2_buf_type */ + __u32 bytesused; + __u32 flags; + __u32 field; /* enum v4l2_field */ + struct old_timeval32 timestamp; struct v4l2_timecode timecode; __u32 sequence; @@ -581,6 +611,31 @@ static int bufsize_v4l2_buffer(struct v4l2_buffer32 __user *p32, u32 *size) return 0; } +static int bufsize_v4l2_buffer_time32(struct v4l2_buffer32_time32 __user *p32, u32 *size) +{ + u32 type; + u32 length; + + if (!access_ok(p32, sizeof(*p32)) || + get_user(type, &p32->type) || + get_user(length, &p32->length)) + return -EFAULT; + + if (V4L2_TYPE_IS_MULTIPLANAR(type)) { + if (length > VIDEO_MAX_PLANES) + return -EINVAL; + + /* + * We don't really care if userspace decides to kill itself + * by passing a very big length value + */ + *size = length * sizeof(struct v4l2_plane); + } else { + *size = 0; + } + return 0; +} + static int get_v4l2_buffer32(struct v4l2_buffer __user *p64, struct v4l2_buffer32 __user *p32, void __user *aux_buf, u32 aux_space) @@ -681,6 +736,106 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *p64, return 0; } +static int get_v4l2_buffer32_time32(struct v4l2_buffer_time32 __user *p64, + struct v4l2_buffer32_time32 __user *p32, + void __user *aux_buf, u32 aux_space) +{ + u32 type; + u32 length; + s32 request_fd; + enum v4l2_memory memory; + struct v4l2_plane32 __user *uplane32; + struct v4l2_plane __user *uplane; + compat_caddr_t p; + int ret; + + if (!access_ok(p32, sizeof(*p32)) || + assign_in_user(&p64->index, &p32->index) || + get_user(type, &p32->type) || + put_user(type, &p64->type) || + assign_in_user(&p64->flags, &p32->flags) || + get_user(memory, &p32->memory) || + put_user(memory, &p64->memory) || + get_user(length, &p32->length) || + put_user(length, &p64->length) || + get_user(request_fd, &p32->request_fd) || + put_user(request_fd, &p64->request_fd)) + return -EFAULT; + + if (V4L2_TYPE_IS_OUTPUT(type)) + if (assign_in_user(&p64->bytesused, &p32->bytesused) || + assign_in_user(&p64->field, &p32->field) || + assign_in_user(&p64->timestamp.tv_sec, + &p32->timestamp.tv_sec) || + assign_in_user(&p64->timestamp.tv_usec, + &p32->timestamp.tv_usec)) + return -EFAULT; + + if (V4L2_TYPE_IS_MULTIPLANAR(type)) { + u32 num_planes = length; + + if (num_planes == 0) { + /* + * num_planes == 0 is legal, e.g. when userspace doesn't + * need planes array on DQBUF + */ + return put_user(NULL, &p64->m.planes); + } + if (num_planes > VIDEO_MAX_PLANES) + return -EINVAL; + + if (get_user(p, &p32->m.planes)) + return -EFAULT; + + uplane32 = compat_ptr(p); + if (!access_ok(uplane32, + num_planes * sizeof(*uplane32))) + return -EFAULT; + + /* + * We don't really care if userspace decides to kill itself + * by passing a very big num_planes value + */ + if (aux_space < num_planes * sizeof(*uplane)) + return -EFAULT; + + uplane = aux_buf; + if (put_user_force(uplane, &p64->m.planes)) + return -EFAULT; + + while (num_planes--) { + ret = get_v4l2_plane32(uplane, uplane32, memory); + if (ret) + return ret; + uplane++; + uplane32++; + } + } else { + switch (memory) { + case V4L2_MEMORY_MMAP: + case V4L2_MEMORY_OVERLAY: + if (assign_in_user(&p64->m.offset, &p32->m.offset)) + return -EFAULT; + break; + case V4L2_MEMORY_USERPTR: { + compat_ulong_t userptr; + + if (get_user(userptr, &p32->m.userptr) || + put_user((unsigned long)compat_ptr(userptr), + &p64->m.userptr)) + return -EFAULT; + break; + } + case V4L2_MEMORY_DMABUF: + if (assign_in_user(&p64->m.fd, &p32->m.fd)) + return -EFAULT; + break; + } + } + + return 0; +} + static int put_v4l2_buffer32(struct v4l2_buffer __user *p64, struct v4l2_buffer32 __user *p32) { @@ -761,6 +916,86 @@ static int put_v4l2_buffer32(struct v4l2_buffer __user *p64, return 0; } +static int put_v4l2_buffer32_time32(struct v4l2_buffer_time32 __user *p64, + struct v4l2_buffer32_time32 __user *p32) +{ + u32 type; + u32 length; + enum v4l2_memory memory; + struct v4l2_plane32 __user *uplane32; + struct v4l2_plane *uplane; + compat_caddr_t p; + int ret; + + if (!access_ok(p32, sizeof(*p32)) || + assign_in_user(&p32->index, &p64->index) || + get_user(type, &p64->type) || + put_user(type, &p32->type) || + assign_in_user(&p32->flags, &p64->flags) || + get_user(memory, &p64->memory) || + put_user(memory, &p32->memory)) + return -EFAULT; + + if (assign_in_user(&p32->bytesused, &p64->bytesused) || + assign_in_user(&p32->field, &p64->field) || + assign_in_user(&p32->timestamp.tv_sec, &p64->timestamp.tv_sec) || + assign_in_user(&p32->timestamp.tv_usec, &p64->timestamp.tv_usec) || + copy_in_user(&p32->timecode, &p64->timecode, sizeof(p64->timecode)) || + assign_in_user(&p32->sequence, &p64->sequence) || + assign_in_user(&p32->reserved2, &p64->reserved2) || + assign_in_user(&p32->request_fd, &p64->request_fd) || + get_user(length, &p64->length) || + put_user(length, &p32->length)) + return -EFAULT; + + if (V4L2_TYPE_IS_MULTIPLANAR(type)) { + u32 num_planes = length; + + if (num_planes == 0) + return 0; + /* We need to define uplane without __user, even though + * it does point to data in userspace here. The reason is + * that v4l2-ioctl.c copies it from userspace to kernelspace, + * so its definition in videodev2.h doesn't have a + * __user markup. Defining uplane with __user causes + * smatch warnings, so instead declare it without __user + * and cast it as a userspace pointer to put_v4l2_plane32(). + */ + if (get_user(uplane, &p64->m.planes)) + return -EFAULT; + if (get_user(p, &p32->m.planes)) + return -EFAULT; + uplane32 = compat_ptr(p); + + while (num_planes--) { + ret = put_v4l2_plane32((void __user *)uplane, + uplane32, memory); + if (ret) + return ret; + ++uplane; + ++uplane32; + } + } else { + switch (memory) { + case V4L2_MEMORY_MMAP: + case V4L2_MEMORY_OVERLAY: + if (assign_in_user(&p32->m.offset, &p64->m.offset)) + return -EFAULT; + break; + case V4L2_MEMORY_USERPTR: + if (assign_in_user(&p32->m.userptr, &p64->m.userptr)) + return -EFAULT; + break; + case V4L2_MEMORY_DMABUF: + if (assign_in_user(&p32->m.fd, &p64->m.fd)) + return -EFAULT; + break; + } + } + + return 0; +} + struct v4l2_framebuffer32 { __u32 capability; __u32 flags; @@ -1028,6 +1263,15 @@ static int put_v4l2_ext_controls32(struct file *file, return 0; } +#ifdef CONFIG_X86_64 +/* + * x86 is the only compat architecture with different struct alignment + * between 32-bit and 64-bit tasks. + * + * On all other architectures, v4l2_event32 and v4l2_event32_time32 are + * the same as v4l2_event and v4l2_event_time32, so we can use the native + * handlers, converting v4l2_event to v4l2_event_time32 if necessary. + */ struct v4l2_event32 { __u32 type; union { @@ -1036,7 +1280,23 @@ struct v4l2_event32 { } u; __u32 pending; __u32 sequence; - struct compat_timespec timestamp; + struct { + compat_s64 tv_sec; + compat_s64 tv_nsec; + } timestamp; + __u32 id; + __u32 reserved[8]; +}; + +struct v4l2_event32_time32 { + __u32 type; + union { + compat_s64 value64; + __u8 data[64]; + } u; + __u32 pending; + __u32 sequence; + struct old_timespec32 timestamp; __u32 id; __u32 reserved[8]; }; @@ -1057,6 +1317,23 @@ static int put_v4l2_event32(struct v4l2_event __user *p64, return 0; } +static int put_v4l2_event32_time32(struct v4l2_event_time32 __user *p64, + struct v4l2_event32_time32 __user *p32) +{ + if (!access_ok(p32, sizeof(*p32)) || + assign_in_user(&p32->type, &p64->type) || + copy_in_user(&p32->u, &p64->u, sizeof(p64->u)) || + assign_in_user(&p32->pending, &p64->pending) || + assign_in_user(&p32->sequence, &p64->sequence) || + assign_in_user(&p32->timestamp.tv_sec, &p64->timestamp.tv_sec) || + assign_in_user(&p32->timestamp.tv_nsec, &p64->timestamp.tv_nsec) || + assign_in_user(&p32->id, &p64->id) || + copy_in_user(p32->reserved, p64->reserved, sizeof(p32->reserved))) + return -EFAULT; + return 0; +} +#endif + struct v4l2_edid32 { __u32 pad; __u32 start_block; @@ -1108,10 +1385,13 @@ static int put_v4l2_edid32(struct v4l2_edid __user *p64, #define VIDIOC_G_FMT32 _IOWR('V', 4, struct v4l2_format32) #define VIDIOC_S_FMT32 _IOWR('V', 5, struct v4l2_format32) #define VIDIOC_QUERYBUF32 _IOWR('V', 9, struct v4l2_buffer32) +#define VIDIOC_QUERYBUF32_TIME32 _IOWR('V', 9, struct v4l2_buffer32_time32) #define VIDIOC_G_FBUF32 _IOR ('V', 10, struct v4l2_framebuffer32) #define VIDIOC_S_FBUF32 _IOW ('V', 11, struct v4l2_framebuffer32) #define VIDIOC_QBUF32 _IOWR('V', 15, struct v4l2_buffer32) +#define VIDIOC_QBUF32_TIME32 _IOWR('V', 15, struct v4l2_buffer32_time32) #define VIDIOC_DQBUF32 _IOWR('V', 17, struct v4l2_buffer32) +#define VIDIOC_DQBUF32_TIME32 _IOWR('V', 17, struct v4l2_buffer32_time32) #define VIDIOC_ENUMSTD32 _IOWR('V', 25, struct v4l2_standard32) #define VIDIOC_ENUMINPUT32 _IOWR('V', 26, struct v4l2_input32) #define VIDIOC_G_EDID32 _IOWR('V', 40, struct v4l2_edid32) @@ -1121,8 +1401,10 @@ static int put_v4l2_edid32(struct v4l2_edid __user *p64, #define VIDIOC_S_EXT_CTRLS32 _IOWR('V', 72, struct v4l2_ext_controls32) #define VIDIOC_TRY_EXT_CTRLS32 _IOWR('V', 73, struct v4l2_ext_controls32) #define VIDIOC_DQEVENT32 _IOR ('V', 89, struct v4l2_event32) +#define VIDIOC_DQEVENT32_TIME32 _IOR ('V', 89, struct v4l2_event32_time32) #define VIDIOC_CREATE_BUFS32 _IOWR('V', 92, struct v4l2_create_buffers32) #define VIDIOC_PREPARE_BUF32 _IOWR('V', 93, struct v4l2_buffer32) +#define VIDIOC_PREPARE_BUF32_TIME32 _IOWR('V', 93, struct v4l2_buffer32_time32) #define VIDIOC_OVERLAY32 _IOW ('V', 14, s32) #define VIDIOC_STREAMON32 _IOW ('V', 18, s32) @@ -1183,36 +1465,45 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar u32 aux_space; int compatible_arg = 1; long err = 0; + unsigned int ncmd; /* * 1. When struct size is different, converts the command. */ switch (cmd) { - case VIDIOC_G_FMT32: cmd = VIDIOC_G_FMT; break; - case VIDIOC_S_FMT32: cmd = VIDIOC_S_FMT; break; - case VIDIOC_QUERYBUF32: cmd = VIDIOC_QUERYBUF; break; - case VIDIOC_G_FBUF32: cmd = VIDIOC_G_FBUF; break; - case VIDIOC_S_FBUF32: cmd = VIDIOC_S_FBUF; break; - case VIDIOC_QBUF32: cmd = VIDIOC_QBUF; break; - case VIDIOC_DQBUF32: cmd = VIDIOC_DQBUF; break; - case VIDIOC_ENUMSTD32: cmd = VIDIOC_ENUMSTD; break; - case VIDIOC_ENUMINPUT32: cmd = VIDIOC_ENUMINPUT; break; - case VIDIOC_TRY_FMT32: cmd = VIDIOC_TRY_FMT; break; - case VIDIOC_G_EXT_CTRLS32: cmd = VIDIOC_G_EXT_CTRLS; break; - case VIDIOC_S_EXT_CTRLS32: cmd = VIDIOC_S_EXT_CTRLS; break; - case VIDIOC_TRY_EXT_CTRLS32: cmd = VIDIOC_TRY_EXT_CTRLS; break; - case VIDIOC_DQEVENT32: cmd = VIDIOC_DQEVENT; break; - case VIDIOC_OVERLAY32: cmd = VIDIOC_OVERLAY; break; - case VIDIOC_STREAMON32: cmd = VIDIOC_STREAMON; break; - case VIDIOC_STREAMOFF32: cmd = VIDIOC_STREAMOFF; break; - case VIDIOC_G_INPUT32: cmd = VIDIOC_G_INPUT; break; - case VIDIOC_S_INPUT32: cmd = VIDIOC_S_INPUT; break; - case VIDIOC_G_OUTPUT32: cmd = VIDIOC_G_OUTPUT; break; - case VIDIOC_S_OUTPUT32: cmd = VIDIOC_S_OUTPUT; break; - case VIDIOC_CREATE_BUFS32: cmd = VIDIOC_CREATE_BUFS; break; - case VIDIOC_PREPARE_BUF32: cmd = VIDIOC_PREPARE_BUF; break; - case VIDIOC_G_EDID32: cmd = VIDIOC_G_EDID; break; - case VIDIOC_S_EDID32: cmd = VIDIOC_S_EDID; break; + case VIDIOC_G_FMT32: ncmd = VIDIOC_G_FMT; break; + case VIDIOC_S_FMT32: ncmd = VIDIOC_S_FMT; break; + case VIDIOC_QUERYBUF32: ncmd = VIDIOC_QUERYBUF; break; + case VIDIOC_QUERYBUF32_TIME32: ncmd = VIDIOC_QUERYBUF_TIME32; break; + case VIDIOC_G_FBUF32: ncmd = VIDIOC_G_FBUF; break; + case VIDIOC_S_FBUF32: ncmd = VIDIOC_S_FBUF; break; + case VIDIOC_QBUF32: ncmd = VIDIOC_QBUF; break; + case VIDIOC_QBUF32_TIME32: ncmd = VIDIOC_QBUF_TIME32; break; + case VIDIOC_DQBUF32: ncmd = VIDIOC_DQBUF; break; + case VIDIOC_DQBUF32_TIME32: ncmd = VIDIOC_DQBUF_TIME32; break; + case VIDIOC_ENUMSTD32: ncmd = VIDIOC_ENUMSTD; break; + case VIDIOC_ENUMINPUT32: ncmd = VIDIOC_ENUMINPUT; break; + case VIDIOC_TRY_FMT32: ncmd = VIDIOC_TRY_FMT; break; + case VIDIOC_G_EXT_CTRLS32: ncmd = VIDIOC_G_EXT_CTRLS; break; + case VIDIOC_S_EXT_CTRLS32: ncmd = VIDIOC_S_EXT_CTRLS; break; + case VIDIOC_TRY_EXT_CTRLS32: ncmd = VIDIOC_TRY_EXT_CTRLS; break; +#ifdef CONFIG_X86_64 + case VIDIOC_DQEVENT32: ncmd = VIDIOC_DQEVENT; break; + case VIDIOC_DQEVENT32_TIME32: ncmd = VIDIOC_DQEVENT_TIME32; break; +#endif + case VIDIOC_OVERLAY32: ncmd = VIDIOC_OVERLAY; break; + case VIDIOC_STREAMON32: ncmd = VIDIOC_STREAMON; break; + case VIDIOC_STREAMOFF32: ncmd = VIDIOC_STREAMOFF; break; + case VIDIOC_G_INPUT32: ncmd = VIDIOC_G_INPUT; break; + case VIDIOC_S_INPUT32: ncmd = VIDIOC_S_INPUT; break; + case VIDIOC_G_OUTPUT32: ncmd = VIDIOC_G_OUTPUT; break; + case VIDIOC_S_OUTPUT32: ncmd = VIDIOC_S_OUTPUT; break; + case VIDIOC_CREATE_BUFS32: ncmd = VIDIOC_CREATE_BUFS; break; + case VIDIOC_PREPARE_BUF32: ncmd = VIDIOC_PREPARE_BUF; break; + case VIDIOC_PREPARE_BUF32_TIME32: ncmd = VIDIOC_PREPARE_BUF_TIME32; break; + case VIDIOC_G_EDID32: ncmd = VIDIOC_G_EDID; break; + case VIDIOC_S_EDID32: ncmd = VIDIOC_S_EDID; break; + default: ncmd = cmd; break; } /* @@ -1221,11 +1512,11 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar * argument into it. */ switch (cmd) { - case VIDIOC_OVERLAY: - case VIDIOC_STREAMON: - case VIDIOC_STREAMOFF: - case VIDIOC_S_INPUT: - case VIDIOC_S_OUTPUT: + case VIDIOC_OVERLAY32: + case VIDIOC_STREAMON32: + case VIDIOC_STREAMOFF32: + case VIDIOC_S_INPUT32: + case VIDIOC_S_OUTPUT32: err = alloc_userspace(sizeof(unsigned int), 0, &new_p64); if (!err && assign_in_user((unsigned int __user *)new_p64, (compat_uint_t __user *)p32)) @@ -1233,23 +1524,23 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar compatible_arg = 0; break; - case VIDIOC_G_INPUT: - case VIDIOC_G_OUTPUT: + case VIDIOC_G_INPUT32: + case VIDIOC_G_OUTPUT32: err = alloc_userspace(sizeof(unsigned int), 0, &new_p64); compatible_arg = 0; break; - case VIDIOC_G_EDID: - case VIDIOC_S_EDID: + case VIDIOC_G_EDID32: + case VIDIOC_S_EDID32: err = alloc_userspace(sizeof(struct v4l2_edid), 0, &new_p64); if (!err) err = get_v4l2_edid32(new_p64, p32); compatible_arg = 0; break; - case VIDIOC_G_FMT: - case VIDIOC_S_FMT: - case VIDIOC_TRY_FMT: + case VIDIOC_G_FMT32: + case VIDIOC_S_FMT32: + case VIDIOC_TRY_FMT32: err = bufsize_v4l2_format(p32, &aux_space); if (!err) err = alloc_userspace(sizeof(struct v4l2_format), @@ -1262,7 +1553,7 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar compatible_arg = 0; break; - case VIDIOC_CREATE_BUFS: + case VIDIOC_CREATE_BUFS32: err = bufsize_v4l2_create(p32, &aux_space); if (!err) err = alloc_userspace(sizeof(struct v4l2_create_buffers), @@ -1275,10 +1566,10 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar compatible_arg = 0; break; - case VIDIOC_PREPARE_BUF: - case VIDIOC_QUERYBUF: - case VIDIOC_QBUF: - case VIDIOC_DQBUF: + case VIDIOC_PREPARE_BUF32: + case VIDIOC_QUERYBUF32: + case VIDIOC_QBUF32: + case VIDIOC_DQBUF32: err = bufsize_v4l2_buffer(p32, &aux_space); if (!err) err = alloc_userspace(sizeof(struct v4l2_buffer), @@ -1291,7 +1582,23 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar compatible_arg = 0; break; - case VIDIOC_S_FBUF: + case VIDIOC_PREPARE_BUF32_TIME32: + case VIDIOC_QUERYBUF32_TIME32: + case VIDIOC_QBUF32_TIME32: + case VIDIOC_DQBUF32_TIME32: + err = bufsize_v4l2_buffer_time32(p32, &aux_space); + if (!err) + err = alloc_userspace(sizeof(struct v4l2_buffer), + aux_space, &new_p64); + if (!err) { + aux_buf = new_p64 + sizeof(struct v4l2_buffer); + err = get_v4l2_buffer32_time32(new_p64, p32, + aux_buf, aux_space); + } + compatible_arg = 0; + break; + + case VIDIOC_S_FBUF32: err = alloc_userspace(sizeof(struct v4l2_framebuffer), 0, &new_p64); if (!err) @@ -1299,13 +1606,13 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar compatible_arg = 0; break; - case VIDIOC_G_FBUF: + case VIDIOC_G_FBUF32: err = alloc_userspace(sizeof(struct v4l2_framebuffer), 0, &new_p64); compatible_arg = 0; break; - case VIDIOC_ENUMSTD: + case VIDIOC_ENUMSTD32: err = alloc_userspace(sizeof(struct v4l2_standard), 0, &new_p64); if (!err) @@ -1313,16 +1620,16 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar compatible_arg = 0; break; - case VIDIOC_ENUMINPUT: + case VIDIOC_ENUMINPUT32: err = alloc_userspace(sizeof(struct v4l2_input), 0, &new_p64); if (!err) err = get_v4l2_input32(new_p64, p32); compatible_arg = 0; break; - case VIDIOC_G_EXT_CTRLS: - case VIDIOC_S_EXT_CTRLS: - case VIDIOC_TRY_EXT_CTRLS: + case VIDIOC_G_EXT_CTRLS32: + case VIDIOC_S_EXT_CTRLS32: + case VIDIOC_TRY_EXT_CTRLS32: err = bufsize_v4l2_ext_controls(p32, &aux_space); if (!err) err = alloc_userspace(sizeof(struct v4l2_ext_controls), @@ -1334,10 +1641,16 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar } compatible_arg = 0; break; - case VIDIOC_DQEVENT: +#ifdef CONFIG_X86_64 + case VIDIOC_DQEVENT32: err = alloc_userspace(sizeof(struct v4l2_event), 0, &new_p64); compatible_arg = 0; break; + case VIDIOC_DQEVENT32_TIME32: + err = alloc_userspace(sizeof(struct v4l2_event_time32), 0, &new_p64); + compatible_arg = 0; + break; +#endif } if (err) return err; @@ -1352,9 +1665,9 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar * Otherwise, it will pass the newly allocated @new_p64 argument. */ if (compatible_arg) - err = native_ioctl(file, cmd, (unsigned long)p32); + err = native_ioctl(file, ncmd, (unsigned long)p32); else - err = native_ioctl(file, cmd, (unsigned long)new_p64); + err = native_ioctl(file, ncmd, (unsigned long)new_p64); if (err == -ENOTTY) return err; @@ -1370,13 +1683,13 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar * the blocks to maximum allowed value. */ switch (cmd) { - case VIDIOC_G_EXT_CTRLS: - case VIDIOC_S_EXT_CTRLS: - case VIDIOC_TRY_EXT_CTRLS: + case VIDIOC_G_EXT_CTRLS32: + case VIDIOC_S_EXT_CTRLS32: + case VIDIOC_TRY_EXT_CTRLS32: if (put_v4l2_ext_controls32(file, new_p64, p32)) err = -EFAULT; break; - case VIDIOC_S_EDID: + case VIDIOC_S_EDID32: if (put_v4l2_edid32(new_p64, p32)) err = -EFAULT; break; @@ -1389,49 +1702,62 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar * the original 32 bits structure. */ switch (cmd) { - case VIDIOC_S_INPUT: - case VIDIOC_S_OUTPUT: - case VIDIOC_G_INPUT: - case VIDIOC_G_OUTPUT: + case VIDIOC_S_INPUT32: + case VIDIOC_S_OUTPUT32: + case VIDIOC_G_INPUT32: + case VIDIOC_G_OUTPUT32: if (assign_in_user((compat_uint_t __user *)p32, ((unsigned int __user *)new_p64))) err = -EFAULT; break; - case VIDIOC_G_FBUF: + case VIDIOC_G_FBUF32: err = put_v4l2_framebuffer32(new_p64, p32); break; - case VIDIOC_DQEVENT: +#ifdef CONFIG_X86_64 + case VIDIOC_DQEVENT32: err = put_v4l2_event32(new_p64, p32); break; - case VIDIOC_G_EDID: + case VIDIOC_DQEVENT32_TIME32: + err = put_v4l2_event32_time32(new_p64, p32); + break; +#endif + + case VIDIOC_G_EDID32: err = put_v4l2_edid32(new_p64, p32); break; - case VIDIOC_G_FMT: - case VIDIOC_S_FMT: - case VIDIOC_TRY_FMT: + case VIDIOC_G_FMT32: + case VIDIOC_S_FMT32: + case VIDIOC_TRY_FMT32: err = put_v4l2_format32(new_p64, p32); break; - case VIDIOC_CREATE_BUFS: + case VIDIOC_CREATE_BUFS32: err = put_v4l2_create32(new_p64, p32); break; - case VIDIOC_PREPARE_BUF: - case VIDIOC_QUERYBUF: - case VIDIOC_QBUF: - case VIDIOC_DQBUF: + case VIDIOC_PREPARE_BUF32: + case VIDIOC_QUERYBUF32: + case VIDIOC_QBUF32: + case VIDIOC_DQBUF32: err = put_v4l2_buffer32(new_p64, p32); break; - case VIDIOC_ENUMSTD: + case VIDIOC_PREPARE_BUF32_TIME32: + case VIDIOC_QUERYBUF32_TIME32: + case VIDIOC_QBUF32_TIME32: + case VIDIOC_DQBUF32_TIME32: + err = put_v4l2_buffer32_time32(new_p64, p32); + break; + + case VIDIOC_ENUMSTD32: err = put_v4l2_standard32(new_p64, p32); break; - case VIDIOC_ENUMINPUT: + case VIDIOC_ENUMINPUT32: err = put_v4l2_input32(new_p64, p32); break; } diff --git a/drivers/media/v4l2-core/v4l2-event.c b/drivers/media/v4l2-core/v4l2-event.c index 9d673d113d7a..290c6b213179 100644 --- a/drivers/media/v4l2-core/v4l2-event.c +++ b/drivers/media/v4l2-core/v4l2-event.c @@ -27,6 +27,7 @@ static unsigned sev_pos(const struct v4l2_subscribed_event *sev, unsigned idx) static int __v4l2_event_dequeue(struct v4l2_fh *fh, struct v4l2_event *event) { struct v4l2_kevent *kev; + struct timespec64 ts; unsigned long flags; spin_lock_irqsave(&fh->vdev->fh_lock, flags); @@ -44,7 +45,9 @@ static int __v4l2_event_dequeue(struct v4l2_fh *fh, struct v4l2_event *event) kev->event.pending = fh->navailable; *event = kev->event; - event->timestamp = ns_to_timespec(kev->ts); + ts = ns_to_timespec64(kev->ts); + event->timestamp.tv_sec = ts.tv_sec; + event->timestamp.tv_nsec = ts.tv_nsec; kev->sev->first = sev_pos(kev->sev, 1); kev->sev->in_use--; diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c index 192cac076761..6ece4320e1d2 100644 --- a/drivers/media/v4l2-core/v4l2-fwnode.c +++ b/drivers/media/v4l2-core/v4l2-fwnode.c @@ -422,7 +422,7 @@ static int __v4l2_fwnode_endpoint_parse(struct fwnode_handle *fwnode, sizeof(*vep) - offsetof(typeof(*vep), bus)); } - pr_debug("===== begin V4L2 endpoint properties\n"); + pr_debug("===== begin parsing endpoint %pfw\n", fwnode); /* * Zero the fwnode graph endpoint memory in case we don't end up parsing @@ -500,7 +500,7 @@ int v4l2_fwnode_endpoint_parse(struct fwnode_handle *fwnode, ret = __v4l2_fwnode_endpoint_parse(fwnode, vep); - pr_debug("===== end V4L2 endpoint properties\n"); + pr_debug("===== end parsing endpoint %pfw\n", fwnode); return ret; } @@ -551,7 +551,7 @@ int v4l2_fwnode_endpoint_alloc_parse(struct fwnode_handle *fwnode, vep->link_frequencies[i]); } - pr_debug("===== end V4L2 endpoint properties\n"); + pr_debug("===== end parsing endpoint %pfw\n", fwnode); return 0; } diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c index 003b7422aeef..aaf83e254272 100644 --- a/drivers/media/v4l2-core/v4l2-ioctl.c +++ b/drivers/media/v4l2-core/v4l2-ioctl.c @@ -474,10 +474,10 @@ static void v4l_print_buffer(const void *arg, bool write_only) const struct v4l2_plane *plane; int i; - pr_cont("%02ld:%02d:%02d.%08ld index=%d, type=%s, request_fd=%d, flags=0x%08x, field=%s, sequence=%d, memory=%s", - p->timestamp.tv_sec / 3600, - (int)(p->timestamp.tv_sec / 60) % 60, - (int)(p->timestamp.tv_sec % 60), + pr_cont("%02d:%02d:%02d.%09ld index=%d, type=%s, request_fd=%d, flags=0x%08x, field=%s, sequence=%d, memory=%s", + (int)p->timestamp.tv_sec / 3600, + ((int)p->timestamp.tv_sec / 60) % 60, + ((int)p->timestamp.tv_sec % 60), (long)p->timestamp.tv_usec, p->index, prt_names(p->type, v4l2_type_names), p->request_fd, @@ -821,7 +821,7 @@ static void v4l_print_event(const void *arg, bool write_only) const struct v4l2_event *p = arg; const struct v4l2_event_ctrl *c; - pr_cont("type=0x%x, pending=%u, sequence=%u, id=%u, timestamp=%lu.%9.9lu\n", + pr_cont("type=0x%x, pending=%u, sequence=%u, id=%u, timestamp=%llu.%9.9llu\n", p->type, p->pending, p->sequence, p->id, p->timestamp.tv_sec, p->timestamp.tv_nsec); switch (p->type) { @@ -961,7 +961,7 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type) return 0; break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: - if (is_vid && is_rx && ops->vidioc_g_fmt_vid_cap_mplane) + if ((is_vid || is_tch) && is_rx && ops->vidioc_g_fmt_vid_cap_mplane) return 0; break; case V4L2_BUF_TYPE_VIDEO_OVERLAY: @@ -3023,8 +3023,162 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size, return ret; } +static unsigned int video_translate_cmd(unsigned int cmd) +{ + switch (cmd) { +#ifdef CONFIG_COMPAT_32BIT_TIME + case VIDIOC_DQEVENT_TIME32: + return VIDIOC_DQEVENT; + case VIDIOC_QUERYBUF_TIME32: + return VIDIOC_QUERYBUF; + case VIDIOC_QBUF_TIME32: + return VIDIOC_QBUF; + case VIDIOC_DQBUF_TIME32: + return VIDIOC_DQBUF; + case VIDIOC_PREPARE_BUF_TIME32: + return VIDIOC_PREPARE_BUF; +#endif + } + + return cmd; +} + +static int video_get_user(void __user *arg, void *parg, unsigned int cmd, + bool *always_copy) +{ + unsigned int n = _IOC_SIZE(cmd); + + if (!(_IOC_DIR(cmd) & _IOC_WRITE)) { + /* read-only ioctl */ + memset(parg, 0, n); + return 0; + } + + switch (cmd) { +#ifdef CONFIG_COMPAT_32BIT_TIME + case VIDIOC_QUERYBUF_TIME32: + case VIDIOC_QBUF_TIME32: + case VIDIOC_DQBUF_TIME32: + case VIDIOC_PREPARE_BUF_TIME32: { + struct v4l2_buffer_time32 vb32; + struct v4l2_buffer *vb = parg; + + if (copy_from_user(&vb32, arg, sizeof(vb32))) + return -EFAULT; + + *vb = (struct v4l2_buffer) { + .index = vb32.index, + .type = vb32.type, + .bytesused = vb32.bytesused, + .flags = vb32.flags, + .field = vb32.field, + .timestamp.tv_sec = vb32.timestamp.tv_sec, + .timestamp.tv_usec = vb32.timestamp.tv_usec, + .timecode = vb32.timecode, + .sequence = vb32.sequence, + .memory = vb32.memory, + .m.userptr = vb32.m.userptr, + .length = vb32.length, + .request_fd = vb32.request_fd, + }; + + if (cmd == VIDIOC_QUERYBUF_TIME32) + vb->request_fd = 0; + + break; + } +#endif + default: + /* + * In some cases, only a few fields are used as input, + * i.e. when the app sets "index" and then the driver + * fills in the rest of the structure for the thing + * with that index. We only need to copy up the first + * non-input field. + */ + if (v4l2_is_known_ioctl(cmd)) { + u32 flags = v4l2_ioctls[_IOC_NR(cmd)].flags; + + if (flags & INFO_FL_CLEAR_MASK) + n = (flags & INFO_FL_CLEAR_MASK) >> 16; + *always_copy = flags & INFO_FL_ALWAYS_COPY; + } + + if (copy_from_user(parg, (void __user *)arg, n)) + return -EFAULT; + + /* zero out anything we don't copy from userspace */ + if (n < _IOC_SIZE(cmd)) + memset((u8 *)parg + n, 0, _IOC_SIZE(cmd) - n); + break; + } + + return 0; +} + +static int video_put_user(void __user *arg, void *parg, unsigned int cmd) +{ + if (!(_IOC_DIR(cmd) & _IOC_READ)) + return 0; + + switch (cmd) { +#ifdef CONFIG_COMPAT_32BIT_TIME + case VIDIOC_DQEVENT_TIME32: { + struct v4l2_event *ev = parg; + struct v4l2_event_time32 ev32 = { + .type = ev->type, + .pending = ev->pending, + .sequence = ev->sequence, + .timestamp.tv_sec = ev->timestamp.tv_sec, + .timestamp.tv_nsec = ev->timestamp.tv_nsec, + .id = ev->id, + }; + + memcpy(&ev32.u, &ev->u, sizeof(ev->u)); + memcpy(&ev32.reserved, &ev->reserved, sizeof(ev->reserved)); + + if (copy_to_user(arg, &ev32, sizeof(ev32))) + return -EFAULT; + break; + } + case VIDIOC_QUERYBUF_TIME32: + case VIDIOC_QBUF_TIME32: + case VIDIOC_DQBUF_TIME32: + case VIDIOC_PREPARE_BUF_TIME32: { + struct v4l2_buffer *vb = parg; + struct v4l2_buffer_time32 vb32 = { + .index = vb->index, + .type = vb->type, + .bytesused = vb->bytesused, + .flags = vb->flags, + .field = vb->field, + .timestamp.tv_sec = vb->timestamp.tv_sec, + .timestamp.tv_usec = vb->timestamp.tv_usec, + .timecode = vb->timecode, + .sequence = vb->sequence, + .memory = vb->memory, + .m.userptr = vb->m.userptr, + .length = vb->length, + .request_fd = vb->request_fd, + }; + + if (copy_to_user(arg, &vb32, sizeof(vb32))) + return -EFAULT; + break; + } +#endif + default: + /* Copy results into user buffer */ + if (copy_to_user(arg, parg, _IOC_SIZE(cmd))) + return -EFAULT; + break; + } + + return 0; +} + long -video_usercopy(struct file *file, unsigned int cmd, unsigned long arg, +video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg, v4l2_kioctl func) { char sbuf[128]; @@ -3036,6 +3190,7 @@ video_usercopy(struct file *file, unsigned int cmd, unsigned long arg, size_t array_size = 0; void __user *user_ptr = NULL; void **kernel_ptr = NULL; + unsigned int cmd = video_translate_cmd(orig_cmd); const size_t ioc_size = _IOC_SIZE(cmd); /* Copy arguments into temp kernel buffer */ @@ -3050,35 +3205,10 @@ video_usercopy(struct file *file, unsigned int cmd, unsigned long arg, parg = mbuf; } - err = -EFAULT; - if (_IOC_DIR(cmd) & _IOC_WRITE) { - unsigned int n = ioc_size; - - /* - * In some cases, only a few fields are used as input, - * i.e. when the app sets "index" and then the driver - * fills in the rest of the structure for the thing - * with that index. We only need to copy up the first - * non-input field. - */ - if (v4l2_is_known_ioctl(cmd)) { - u32 flags = v4l2_ioctls[_IOC_NR(cmd)].flags; - - if (flags & INFO_FL_CLEAR_MASK) - n = (flags & INFO_FL_CLEAR_MASK) >> 16; - always_copy = flags & INFO_FL_ALWAYS_COPY; - } - - if (copy_from_user(parg, (void __user *)arg, n)) - goto out; - - /* zero out anything we don't copy from userspace */ - if (n < ioc_size) - memset((u8 *)parg + n, 0, ioc_size - n); - } else { - /* read-only ioctl */ - memset(parg, 0, ioc_size); - } + err = video_get_user((void __user *)arg, parg, orig_cmd, + &always_copy); + if (err) + goto out; } err = check_array_args(cmd, parg, &array_size, &user_ptr, &kernel_ptr); @@ -3131,15 +3261,8 @@ video_usercopy(struct file *file, unsigned int cmd, unsigned long arg, goto out; out_array_args: - /* Copy results into user buffer */ - switch (_IOC_DIR(cmd)) { - case _IOC_READ: - case (_IOC_WRITE | _IOC_READ): - if (copy_to_user((void __user *)arg, parg, ioc_size)) - err = -EFAULT; - break; - } - + if (video_put_user((void __user *)arg, parg, orig_cmd)) + err = -EFAULT; out: kvfree(mbuf); return err; diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c index 9e987c0f840e..a376b351135f 100644 --- a/drivers/media/v4l2-core/v4l2-subdev.c +++ b/drivers/media/v4l2-core/v4l2-subdev.c @@ -331,8 +331,8 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) struct v4l2_fh *vfh = file->private_data; #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API) struct v4l2_subdev_fh *subdev_fh = to_v4l2_subdev_fh(vfh); - int rval; #endif + int rval; switch (cmd) { case VIDIOC_QUERYCTRL: @@ -392,6 +392,30 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) return v4l2_event_dequeue(vfh, arg, file->f_flags & O_NONBLOCK); + case VIDIOC_DQEVENT_TIME32: { + struct v4l2_event_time32 *ev32 = arg; + struct v4l2_event ev = { }; + + if (!(sd->flags & V4L2_SUBDEV_FL_HAS_EVENTS)) + return -ENOIOCTLCMD; + + rval = v4l2_event_dequeue(vfh, &ev, file->f_flags & O_NONBLOCK); + + *ev32 = (struct v4l2_event_time32) { + .type = ev.type, + .pending = ev.pending, + .sequence = ev.sequence, + .timestamp.tv_sec = ev.timestamp.tv_sec, + .timestamp.tv_nsec = ev.timestamp.tv_nsec, + .id = ev.id, + }; + + memcpy(&ev32->u, &ev.u, sizeof(ev.u)); + memcpy(&ev32->reserved, &ev.reserved, sizeof(ev.reserved)); + + return rval; + } + case VIDIOC_SUBSCRIBE_EVENT: return v4l2_subdev_call(sd, core, subscribe_event, vfh, arg); diff --git a/drivers/media/v4l2-core/videobuf-core.c b/drivers/media/v4l2-core/videobuf-core.c index 939fc11cf080..2686f03b322e 100644 --- a/drivers/media/v4l2-core/videobuf-core.c +++ b/drivers/media/v4l2-core/videobuf-core.c @@ -19,6 +19,7 @@ #include <linux/interrupt.h> #include <media/videobuf-core.h> +#include <media/v4l2-common.h> #define MAGIC_BUFFER 0x20070728 #define MAGIC_CHECK(is, should) \ @@ -364,7 +365,7 @@ static void videobuf_status(struct videobuf_queue *q, struct v4l2_buffer *b, } b->field = vb->field; - b->timestamp = ns_to_timeval(vb->ts); + v4l2_buffer_set_timestamp(b, vb->ts); b->bytesused = vb->size; b->sequence = vb->field_count >> 1; } @@ -578,7 +579,7 @@ int videobuf_qbuf(struct videobuf_queue *q, struct v4l2_buffer *b) || q->type == V4L2_BUF_TYPE_SDR_OUTPUT) { buf->size = b->bytesused; buf->field = b->field; - buf->ts = v4l2_timeval_to_ns(&b->timestamp); + buf->ts = v4l2_buffer_get_timestamp(b); } break; case V4L2_MEMORY_USERPTR: diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 7a4a41dddc97..2b203290e7b9 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -758,6 +758,7 @@ config MFD_MAX77650 depends on OF || COMPILE_TEST select MFD_CORE select REGMAP_I2C + select REGMAP_IRQ help Say Y here to add support for Maxim Semiconductor MAX77650 and MAX77651 Power Management ICs. This is the core multifunction @@ -1065,7 +1066,7 @@ config MFD_RN5T618 functionality of the device. config MFD_SEC_CORE - tristate "SAMSUNG Electronics PMIC Series Support" + tristate "Samsung Electronics PMIC Series Support" depends on I2C=y select MFD_CORE select REGMAP_I2C @@ -1906,6 +1907,21 @@ config MFD_ROHM_BD70528 10 bits SAR ADC for battery temperature monitor and 1S battery charger. +config MFD_ROHM_BD71828 + tristate "ROHM BD71828 Power Management IC" + depends on I2C=y + depends on OF + select REGMAP_I2C + select REGMAP_IRQ + select MFD_CORE + help + Select this option to get support for the ROHM BD71828 Power + Management IC. BD71828GW is a single-chip power management IC for + battery-powered portable devices. The IC integrates 7 buck + converters, 7 LDOs, and a 1500 mA single-cell linear charger. + Also included is a Coulomb counter, a real-time clock (RTC), and + a 32.768 kHz clock gate. + config MFD_STM32_LPTIMER tristate "Support for STM32 Low-Power Timer" depends on (ARCH_STM32 && OF) || COMPILE_TEST @@ -1960,6 +1976,18 @@ config MFD_STMFX additional drivers must be enabled in order to use the functionality of the device. +config MFD_WCD934X + tristate "Support for WCD9340/WCD9341 Codec" + depends on SLIMBUS + select REGMAP + select REGMAP_SLIMBUS + select REGMAP_IRQ + select MFD_CORE + help + Support for the Qualcomm WCD9340/WCD9341 Codec. + This driver provides common support WCD934x audio codec and its + associated Pin Controller, Soundwire Controller and Audio codec. + menu "Multimedia Capabilities Port drivers" depends on ARCH_SA1100 diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 5fe930c76ade..b83f172545e1 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -58,6 +58,7 @@ endif ifeq ($(CONFIG_MFD_CS47L24),y) obj-$(CONFIG_MFD_ARIZONA) += cs47l24-tables.o endif +obj-$(CONFIG_MFD_WCD934X) += wcd934x.o obj-$(CONFIG_MFD_WM8400) += wm8400-core.o wm831x-objs := wm831x-core.o wm831x-irq.o wm831x-otp.o wm831x-objs += wm831x-auxadc.o @@ -252,6 +253,7 @@ obj-$(CONFIG_MFD_MXS_LRADC) += mxs-lradc.o obj-$(CONFIG_MFD_SC27XX_PMIC) += sprd-sc27xx-spi.o obj-$(CONFIG_RAVE_SP_CORE) += rave-sp.o obj-$(CONFIG_MFD_ROHM_BD70528) += rohm-bd70528.o +obj-$(CONFIG_MFD_ROHM_BD71828) += rohm-bd71828.o obj-$(CONFIG_MFD_ROHM_BD718XX) += rohm-bd718x7.o obj-$(CONFIG_MFD_STMFX) += stmfx.o diff --git a/drivers/mfd/ab8500-core.c b/drivers/mfd/ab8500-core.c index bafc729fc434..a3bac9da8cbb 100644 --- a/drivers/mfd/ab8500-core.c +++ b/drivers/mfd/ab8500-core.c @@ -631,8 +631,8 @@ static const struct mfd_cell ab8500_devs[] = { NULL, NULL, 0, 0, "stericsson,ab8500-ext-regulator"), OF_MFD_CELL("ab8500-regulator", NULL, NULL, 0, 0, "stericsson,ab8500-regulator"), - OF_MFD_CELL("abx500-clk", - NULL, NULL, 0, 0, "stericsson,abx500-clk"), + OF_MFD_CELL("ab8500-clk", + NULL, NULL, 0, 0, "stericsson,ab8500-clk"), OF_MFD_CELL("ab8500-gpadc", NULL, NULL, 0, 0, "stericsson,ab8500-gpadc"), OF_MFD_CELL("ab8500-rtc", @@ -718,17 +718,20 @@ static const struct mfd_cell ab8505_devs[] = { #ifdef CONFIG_DEBUG_FS { .name = "ab8500-debug", + .of_compatible = "stericsson,ab8500-debug", }, #endif { .name = "ab8500-sysctrl", + .of_compatible = "stericsson,ab8500-sysctrl", }, { .name = "ab8500-regulator", + .of_compatible = "stericsson,ab8505-regulator", }, { .name = "abx500-clk", - .of_compatible = "stericsson,abx500-clk", + .of_compatible = "stericsson,ab8500-clk", }, { .name = "ab8500-gpadc", @@ -736,25 +739,32 @@ static const struct mfd_cell ab8505_devs[] = { }, { .name = "ab8500-rtc", + .of_compatible = "stericsson,ab8500-rtc", }, { .name = "ab8500-acc-det", + .of_compatible = "stericsson,ab8500-acc-det", }, { .name = "ab8500-poweron-key", + .of_compatible = "stericsson,ab8500-poweron-key", }, { .name = "ab8500-pwm", + .of_compatible = "stericsson,ab8500-pwm", .id = 1, }, { .name = "pinctrl-ab8505", + .of_compatible = "stericsson,ab8505-gpio", }, { .name = "ab8500-usb", + .of_compatible = "stericsson,ab8500-usb", }, { .name = "ab8500-codec", + .of_compatible = "stericsson,ab8500-codec", }, { .name = "ab-iddet", @@ -1276,7 +1286,7 @@ static int ab8500_probe(struct platform_device *pdev) static const struct platform_device_id ab8500_id[] = { { "ab8500-core", AB8500_VERSION_AB8500 }, - { "ab8505-i2c", AB8500_VERSION_AB8505 }, + { "ab8505-core", AB8500_VERSION_AB8505 }, { "ab9540-i2c", AB8500_VERSION_AB9540 }, { "ab8540-i2c", AB8500_VERSION_AB8540 }, { } diff --git a/drivers/mfd/atmel-hlcdc.c b/drivers/mfd/atmel-hlcdc.c index 64013c57a920..3c2414ba4b01 100644 --- a/drivers/mfd/atmel-hlcdc.c +++ b/drivers/mfd/atmel-hlcdc.c @@ -19,6 +19,7 @@ struct atmel_hlcdc_regmap { void __iomem *regs; + struct device *dev; }; static const struct mfd_cell atmel_hlcdc_cells[] = { @@ -39,10 +40,17 @@ static int regmap_atmel_hlcdc_reg_write(void *context, unsigned int reg, if (reg <= ATMEL_HLCDC_DIS) { u32 status; - - readl_poll_timeout_atomic(hregmap->regs + ATMEL_HLCDC_SR, - status, !(status & ATMEL_HLCDC_SIP), - 1, 100); + int ret; + + ret = readl_poll_timeout_atomic(hregmap->regs + ATMEL_HLCDC_SR, + status, + !(status & ATMEL_HLCDC_SIP), + 1, 100); + if (ret) { + dev_err(hregmap->dev, + "Timeout! Clock domain synchronization is in progress!\n"); + return ret; + } } writel(val, hregmap->regs + reg); @@ -90,6 +98,8 @@ static int atmel_hlcdc_probe(struct platform_device *pdev) if (IS_ERR(hregmap->regs)) return PTR_ERR(hregmap->regs); + hregmap->dev = &pdev->dev; + hlcdc->irq = platform_get_irq(pdev, 0); if (hlcdc->irq < 0) return hlcdc->irq; diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c index a4aaadaa0cb0..aa59496e4376 100644 --- a/drivers/mfd/axp20x.c +++ b/drivers/mfd/axp20x.c @@ -126,7 +126,7 @@ static const struct regmap_range axp288_writeable_ranges[] = { static const struct regmap_range axp288_volatile_ranges[] = { regmap_reg_range(AXP20X_PWR_INPUT_STATUS, AXP288_POWER_REASON), regmap_reg_range(AXP288_BC_GLOBAL, AXP288_BC_GLOBAL), - regmap_reg_range(AXP288_BC_DET_STAT, AXP288_BC_DET_STAT), + regmap_reg_range(AXP288_BC_DET_STAT, AXP20X_VBUS_IPSOUT_MGMT), regmap_reg_range(AXP20X_CHRG_BAK_CTRL, AXP20X_CHRG_BAK_CTRL), regmap_reg_range(AXP20X_IRQ1_EN, AXP20X_IPSOUT_V_HIGH_L), regmap_reg_range(AXP20X_TIMER_CTRL, AXP20X_TIMER_CTRL), diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c index c4b977a5dd96..39e611695053 100644 --- a/drivers/mfd/cros_ec_dev.c +++ b/drivers/mfd/cros_ec_dev.c @@ -5,8 +5,8 @@ * Copyright (C) 2014 Google, Inc. */ +#include <linux/kconfig.h> #include <linux/mfd/core.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/mod_devicetable.h> #include <linux/of_platform.h> @@ -87,6 +87,10 @@ static const struct mfd_cell cros_usbpd_charger_cells[] = { { .name = "cros-usbpd-logger", }, }; +static const struct mfd_cell cros_usbpd_notify_cells[] = { + { .name = "cros-usbpd-notify", }, +}; + static const struct cros_feature_to_cells cros_subdevices[] = { { .id = EC_FEATURE_CEC, @@ -203,6 +207,23 @@ static int ec_device_probe(struct platform_device *pdev) } /* + * The PD notifier driver cell is separate since it only needs to be + * explicitly added on platforms that don't have the PD notifier ACPI + * device entry defined. + */ + if (IS_ENABLED(CONFIG_OF)) { + if (cros_ec_check_features(ec, EC_FEATURE_USB_PD)) { + retval = mfd_add_hotplug_devices(ec->dev, + cros_usbpd_notify_cells, + ARRAY_SIZE(cros_usbpd_notify_cells)); + if (retval) + dev_err(ec->dev, + "failed to add PD notify devices: %d\n", + retval); + } + } + + /* * The following subdevices cannot be detected by sending the * EC_FEATURE_GET_CMD to the Embedded Controller device. */ diff --git a/drivers/mfd/cs47l15-tables.c b/drivers/mfd/cs47l15-tables.c index f81b45336690..3c77f0a24e9b 100644 --- a/drivers/mfd/cs47l15-tables.c +++ b/drivers/mfd/cs47l15-tables.c @@ -112,6 +112,7 @@ static const struct reg_default cs47l15_reg_default[] = { { 0x000001dd, 0x0011 }, /* R477 (0x1DD) - FLL AO Control 11 */ { 0x00000218, 0x00e6 }, /* R536 (0x218) - Mic Bias Ctrl 1 */ { 0x0000021c, 0x0222 }, /* R540 (0x21C) - Mic Bias Ctrl 5 */ + { 0x00000293, 0x0080 }, /* R659 (0x293) - Accessory Detect Mode 1 */ { 0x00000299, 0x0000 }, /* R665 (0x299) - Headphone Detect 0 */ { 0x0000029b, 0x0000 }, /* R667 (0x29B) - Headphone Detect 1 */ { 0x000002a2, 0x0010 }, /* R674 (0x2A2) - Mic Detect 1 Control 0 */ diff --git a/drivers/mfd/da9062-core.c b/drivers/mfd/da9062-core.c index e69626867c26..419c73533401 100644 --- a/drivers/mfd/da9062-core.c +++ b/drivers/mfd/da9062-core.c @@ -233,6 +233,14 @@ static struct resource da9062_onkey_resources[] = { DEFINE_RES_NAMED(DA9062_IRQ_ONKEY, 1, "ONKEY", IORESOURCE_IRQ), }; +static struct resource da9062_gpio_resources[] = { + DEFINE_RES_NAMED(DA9062_IRQ_GPI0, 1, "GPI0", IORESOURCE_IRQ), + DEFINE_RES_NAMED(DA9062_IRQ_GPI1, 1, "GPI1", IORESOURCE_IRQ), + DEFINE_RES_NAMED(DA9062_IRQ_GPI2, 1, "GPI2", IORESOURCE_IRQ), + DEFINE_RES_NAMED(DA9062_IRQ_GPI3, 1, "GPI3", IORESOURCE_IRQ), + DEFINE_RES_NAMED(DA9062_IRQ_GPI4, 1, "GPI4", IORESOURCE_IRQ), +}; + static const struct mfd_cell da9062_devs[] = { { .name = "da9062-core", @@ -248,7 +256,7 @@ static const struct mfd_cell da9062_devs[] = { .name = "da9062-watchdog", .num_resources = ARRAY_SIZE(da9062_wdt_resources), .resources = da9062_wdt_resources, - .of_compatible = "dlg,da9062-wdt", + .of_compatible = "dlg,da9062-watchdog", }, { .name = "da9062-thermal", @@ -266,7 +274,13 @@ static const struct mfd_cell da9062_devs[] = { .name = "da9062-onkey", .num_resources = ARRAY_SIZE(da9062_onkey_resources), .resources = da9062_onkey_resources, - .of_compatible = "dlg,da9062-onkey", + .of_compatible = "dlg,da9062-onkey", + }, + { + .name = "da9062-gpio", + .num_resources = ARRAY_SIZE(da9062_gpio_resources), + .resources = da9062_gpio_resources, + .of_compatible = "dlg,da9062-gpio", }, }; diff --git a/drivers/mfd/db8500-prcmu.c b/drivers/mfd/db8500-prcmu.c index 57ac58b4b5f3..0452b43b0423 100644 --- a/drivers/mfd/db8500-prcmu.c +++ b/drivers/mfd/db8500-prcmu.c @@ -542,102 +542,6 @@ static struct dsiescclk dsiescclk[3] = { } }; - -/* -* Used by MCDE to setup all necessary PRCMU registers -*/ -#define PRCMU_RESET_DSIPLL 0x00004000 -#define PRCMU_UNCLAMP_DSIPLL 0x00400800 - -#define PRCMU_CLK_PLL_DIV_SHIFT 0 -#define PRCMU_CLK_PLL_SW_SHIFT 5 -#define PRCMU_CLK_38 (1 << 9) -#define PRCMU_CLK_38_SRC (1 << 10) -#define PRCMU_CLK_38_DIV (1 << 11) - -/* PLLDIV=12, PLLSW=4 (PLLDDR) */ -#define PRCMU_DSI_CLOCK_SETTING 0x0000008C - -/* DPI 50000000 Hz */ -#define PRCMU_DPI_CLOCK_SETTING ((1 << PRCMU_CLK_PLL_SW_SHIFT) | \ - (16 << PRCMU_CLK_PLL_DIV_SHIFT)) -#define PRCMU_DSI_LP_CLOCK_SETTING 0x00000E00 - -/* D=101, N=1, R=4, SELDIV2=0 */ -#define PRCMU_PLLDSI_FREQ_SETTING 0x00040165 - -#define PRCMU_ENABLE_PLLDSI 0x00000001 -#define PRCMU_DISABLE_PLLDSI 0x00000000 -#define PRCMU_RELEASE_RESET_DSS 0x0000400C -#define PRCMU_DSI_PLLOUT_SEL_SETTING 0x00000202 -/* ESC clk, div0=1, div1=1, div2=3 */ -#define PRCMU_ENABLE_ESCAPE_CLOCK_DIV 0x07030101 -#define PRCMU_DISABLE_ESCAPE_CLOCK_DIV 0x00030101 -#define PRCMU_DSI_RESET_SW 0x00000007 - -#define PRCMU_PLLDSI_LOCKP_LOCKED 0x3 - -int db8500_prcmu_enable_dsipll(void) -{ - int i; - - /* Clear DSIPLL_RESETN */ - writel(PRCMU_RESET_DSIPLL, PRCM_APE_RESETN_CLR); - /* Unclamp DSIPLL in/out */ - writel(PRCMU_UNCLAMP_DSIPLL, PRCM_MMIP_LS_CLAMP_CLR); - - /* Set DSI PLL FREQ */ - writel(PRCMU_PLLDSI_FREQ_SETTING, PRCM_PLLDSI_FREQ); - writel(PRCMU_DSI_PLLOUT_SEL_SETTING, PRCM_DSI_PLLOUT_SEL); - /* Enable Escape clocks */ - writel(PRCMU_ENABLE_ESCAPE_CLOCK_DIV, PRCM_DSITVCLK_DIV); - - /* Start DSI PLL */ - writel(PRCMU_ENABLE_PLLDSI, PRCM_PLLDSI_ENABLE); - /* Reset DSI PLL */ - writel(PRCMU_DSI_RESET_SW, PRCM_DSI_SW_RESET); - for (i = 0; i < 10; i++) { - if ((readl(PRCM_PLLDSI_LOCKP) & PRCMU_PLLDSI_LOCKP_LOCKED) - == PRCMU_PLLDSI_LOCKP_LOCKED) - break; - udelay(100); - } - /* Set DSIPLL_RESETN */ - writel(PRCMU_RESET_DSIPLL, PRCM_APE_RESETN_SET); - return 0; -} - -int db8500_prcmu_disable_dsipll(void) -{ - /* Disable dsi pll */ - writel(PRCMU_DISABLE_PLLDSI, PRCM_PLLDSI_ENABLE); - /* Disable escapeclock */ - writel(PRCMU_DISABLE_ESCAPE_CLOCK_DIV, PRCM_DSITVCLK_DIV); - return 0; -} - -int db8500_prcmu_set_display_clocks(void) -{ - unsigned long flags; - - spin_lock_irqsave(&clk_mgt_lock, flags); - - /* Grab the HW semaphore. */ - while ((readl(PRCM_SEM) & PRCM_SEM_PRCM_SEM) != 0) - cpu_relax(); - - writel(PRCMU_DSI_CLOCK_SETTING, prcmu_base + PRCM_HDMICLK_MGT); - writel(PRCMU_DSI_LP_CLOCK_SETTING, prcmu_base + PRCM_TVCLK_MGT); - writel(PRCMU_DPI_CLOCK_SETTING, prcmu_base + PRCM_LCDCLK_MGT); - - /* Release the HW semaphore. */ - writel(0, PRCM_SEM); - - spin_unlock_irqrestore(&clk_mgt_lock, flags); - - return 0; -} - u32 db8500_prcmu_read(unsigned int reg) { return readl(prcmu_base + reg); @@ -3060,30 +2964,44 @@ static const struct mfd_cell db8500_prcmu_devs[] = { static int db8500_prcmu_register_ab8500(struct device *parent) { struct device_node *np; - struct resource ab8500_resource; + struct resource ab850x_resource; const struct mfd_cell ab8500_cell = { .name = "ab8500-core", .of_compatible = "stericsson,ab8500", .id = AB8500_VERSION_AB8500, - .resources = &ab8500_resource, + .resources = &ab850x_resource, .num_resources = 1, }; + const struct mfd_cell ab8505_cell = { + .name = "ab8505-core", + .of_compatible = "stericsson,ab8505", + .id = AB8500_VERSION_AB8505, + .resources = &ab850x_resource, + .num_resources = 1, + }; + const struct mfd_cell *ab850x_cell; if (!parent->of_node) return -ENODEV; /* Look up the device node, sneak the IRQ out of it */ for_each_child_of_node(parent->of_node, np) { - if (of_device_is_compatible(np, ab8500_cell.of_compatible)) + if (of_device_is_compatible(np, ab8500_cell.of_compatible)) { + ab850x_cell = &ab8500_cell; break; + } + if (of_device_is_compatible(np, ab8505_cell.of_compatible)) { + ab850x_cell = &ab8505_cell; + break; + } } if (!np) { - dev_info(parent, "could not find AB8500 node in the device tree\n"); + dev_info(parent, "could not find AB850X node in the device tree\n"); return -ENODEV; } - of_irq_to_resource_table(np, &ab8500_resource, 1); + of_irq_to_resource_table(np, &ab850x_resource, 1); - return mfd_add_devices(parent, 0, &ab8500_cell, 1, NULL, 0, NULL); + return mfd_add_devices(parent, 0, ab850x_cell, 1, NULL, 0, NULL); } /** diff --git a/drivers/mfd/dln2.c b/drivers/mfd/dln2.c index 381593fbe50f..7841c11411d0 100644 --- a/drivers/mfd/dln2.c +++ b/drivers/mfd/dln2.c @@ -722,6 +722,8 @@ static int dln2_probe(struct usb_interface *interface, const struct usb_device_id *usb_id) { struct usb_host_interface *hostif = interface->cur_altsetting; + struct usb_endpoint_descriptor *epin; + struct usb_endpoint_descriptor *epout; struct device *dev = &interface->dev; struct dln2_dev *dln2; int ret; @@ -731,12 +733,19 @@ static int dln2_probe(struct usb_interface *interface, hostif->desc.bNumEndpoints < 2) return -ENODEV; + epin = &hostif->endpoint[0].desc; + epout = &hostif->endpoint[1].desc; + if (!usb_endpoint_is_bulk_out(epout)) + return -ENODEV; + if (!usb_endpoint_is_bulk_in(epin)) + return -ENODEV; + dln2 = kzalloc(sizeof(*dln2), GFP_KERNEL); if (!dln2) return -ENOMEM; - dln2->ep_out = hostif->endpoint[0].desc.bEndpointAddress; - dln2->ep_in = hostif->endpoint[1].desc.bEndpointAddress; + dln2->ep_out = epout->bEndpointAddress; + dln2->ep_in = epin->bEndpointAddress; dln2->usb_dev = usb_get_dev(interface_to_usbdev(interface)); dln2->interface = interface; usb_set_intfdata(interface, dln2); diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index b33030e3385c..c40a6c7d0cf8 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -240,6 +240,19 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x4b79), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x4b7a), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x4b7b), (kernel_ulong_t)&bxt_i2c_info }, + /* JSL */ + { PCI_VDEVICE(INTEL, 0x4da8), (kernel_ulong_t)&spt_uart_info }, + { PCI_VDEVICE(INTEL, 0x4da9), (kernel_ulong_t)&spt_uart_info }, + { PCI_VDEVICE(INTEL, 0x4daa), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x4dab), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x4daf), (kernel_ulong_t)&spt_uart_info }, + { PCI_VDEVICE(INTEL, 0x4dc5), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4dc6), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4de8), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4de9), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4dea), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4deb), (kernel_ulong_t)&bxt_i2c_info }, + { PCI_VDEVICE(INTEL, 0x4dfb), (kernel_ulong_t)&spt_info }, /* APL */ { PCI_VDEVICE(INTEL, 0x5aac), (kernel_ulong_t)&apl_i2c_info }, { PCI_VDEVICE(INTEL, 0x5aae), (kernel_ulong_t)&apl_i2c_info }, diff --git a/drivers/mfd/madera-core.c b/drivers/mfd/madera-core.c index a8cfadc1fc01..7e0835cb062b 100644 --- a/drivers/mfd/madera-core.c +++ b/drivers/mfd/madera-core.c @@ -35,6 +35,9 @@ #define MADERA_32KZ_MCLK2 1 +#define MADERA_RESET_MIN_US 2000 +#define MADERA_RESET_MAX_US 3000 + static const char * const madera_core_supplies[] = { "AVDD", "DBVDD1", @@ -199,7 +202,7 @@ EXPORT_SYMBOL_GPL(madera_name_from_type); #define MADERA_BOOT_POLL_INTERVAL_USEC 5000 #define MADERA_BOOT_POLL_TIMEOUT_USEC 25000 -static int madera_wait_for_boot(struct madera *madera) +static int madera_wait_for_boot_noack(struct madera *madera) { ktime_t timeout; unsigned int val = 0; @@ -226,6 +229,13 @@ static int madera_wait_for_boot(struct madera *madera) ret = -ETIMEDOUT; } + return ret; +} + +static int madera_wait_for_boot(struct madera *madera) +{ + int ret = madera_wait_for_boot_noack(madera); + /* * BOOT_DONE defaults to unmasked on boot so we must ack it. * Do this even after a timeout to avoid interrupt storms. @@ -249,16 +259,13 @@ static int madera_soft_reset(struct madera *madera) } /* Allow time for internal clocks to startup after reset */ - usleep_range(1000, 2000); + usleep_range(MADERA_RESET_MIN_US, MADERA_RESET_MAX_US); return 0; } static void madera_enable_hard_reset(struct madera *madera) { - if (!madera->pdata.reset) - return; - /* * There are many existing out-of-tree users of these codecs that we * can't break so preserve the expected behaviour of setting the line @@ -269,11 +276,9 @@ static void madera_enable_hard_reset(struct madera *madera) static void madera_disable_hard_reset(struct madera *madera) { - if (!madera->pdata.reset) - return; - gpiod_set_raw_value_cansleep(madera->pdata.reset, 1); - usleep_range(1000, 2000); + + usleep_range(MADERA_RESET_MIN_US, MADERA_RESET_MAX_US); } static int __maybe_unused madera_runtime_resume(struct device *dev) @@ -292,6 +297,8 @@ static int __maybe_unused madera_runtime_resume(struct device *dev) regcache_cache_only(madera->regmap, false); regcache_cache_only(madera->regmap_32bit, false); + usleep_range(MADERA_RESET_MIN_US, MADERA_RESET_MAX_US); + ret = madera_wait_for_boot(madera); if (ret) goto err; @@ -545,6 +552,12 @@ int madera_dev_init(struct madera *madera) regcache_cache_only(madera->regmap, false); regcache_cache_only(madera->regmap_32bit, false); + ret = madera_wait_for_boot_noack(madera); + if (ret) { + dev_err(madera->dev, "Device failed initial boot: %d\n", ret); + goto err_reset; + } + /* * Now we can power up and verify that this is a chip we know about * before we start doing any writes to its registers. @@ -650,7 +663,7 @@ int madera_dev_init(struct madera *madera) ret = madera_wait_for_boot(madera); if (ret) { - dev_err(madera->dev, "Device failed initial boot: %d\n", ret); + dev_err(madera->dev, "Failed to clear boot done: %d\n", ret); goto err_reset; } diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c index da5cd9c92a59..ead2e79036a9 100644 --- a/drivers/mfd/rn5t618.c +++ b/drivers/mfd/rn5t618.c @@ -26,6 +26,7 @@ static bool rn5t618_volatile_reg(struct device *dev, unsigned int reg) case RN5T618_WATCHDOGCNT: case RN5T618_DCIRQ: case RN5T618_ILIMDATAH ... RN5T618_AIN0DATAL: + case RN5T618_ADCCNT3: case RN5T618_IR_ADC1 ... RN5T618_IR_ADC3: case RN5T618_IR_GPR: case RN5T618_IR_GPF: diff --git a/drivers/mfd/rohm-bd70528.c b/drivers/mfd/rohm-bd70528.c index ef6786fd3b00..5c44d3b77b3e 100644 --- a/drivers/mfd/rohm-bd70528.c +++ b/drivers/mfd/rohm-bd70528.c @@ -48,7 +48,7 @@ static struct mfd_cell bd70528_mfd_cells[] = { * We use BD71837 driver to drive the clock block. Only differences to * BD70528 clock gate are the register address and mask. */ - { .name = "bd718xx-clk", }, + { .name = "bd70528-clk", }, { .name = "bd70528-wdt", }, { .name = "bd70528-power", @@ -236,7 +236,6 @@ static int bd70528_i2c_probe(struct i2c_client *i2c, dev_set_drvdata(&i2c->dev, &bd70528->chip); - bd70528->chip.chip_type = ROHM_CHIP_TYPE_BD70528; bd70528->chip.regmap = devm_regmap_init_i2c(i2c, &bd70528_regmap); if (IS_ERR(bd70528->chip.regmap)) { dev_err(&i2c->dev, "Failed to initialize Regmap\n"); diff --git a/drivers/mfd/rohm-bd71828.c b/drivers/mfd/rohm-bd71828.c new file mode 100644 index 000000000000..210261d026f2 --- /dev/null +++ b/drivers/mfd/rohm-bd71828.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Copyright (C) 2019 ROHM Semiconductors +// +// ROHM BD71828 PMIC driver + +#include <linux/gpio_keys.h> +#include <linux/i2c.h> +#include <linux/input.h> +#include <linux/interrupt.h> +#include <linux/ioport.h> +#include <linux/irq.h> +#include <linux/mfd/core.h> +#include <linux/mfd/rohm-bd71828.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/regmap.h> +#include <linux/types.h> + +static struct gpio_keys_button button = { + .code = KEY_POWER, + .gpio = -1, + .type = EV_KEY, +}; + +static struct gpio_keys_platform_data bd71828_powerkey_data = { + .buttons = &button, + .nbuttons = 1, + .name = "bd71828-pwrkey", +}; + +static const struct resource rtc_irqs[] = { + DEFINE_RES_IRQ_NAMED(BD71828_INT_RTC0, "bd71828-rtc-alm-0"), + DEFINE_RES_IRQ_NAMED(BD71828_INT_RTC1, "bd71828-rtc-alm-1"), + DEFINE_RES_IRQ_NAMED(BD71828_INT_RTC2, "bd71828-rtc-alm-2"), +}; + +static struct mfd_cell bd71828_mfd_cells[] = { + { .name = "bd71828-pmic", }, + { .name = "bd71828-gpio", }, + { .name = "bd71828-led", .of_compatible = "rohm,bd71828-leds" }, + /* + * We use BD71837 driver to drive the clock block. Only differences to + * BD70528 clock gate are the register address and mask. + */ + { .name = "bd71828-clk", }, + { .name = "bd71827-power", }, + { + .name = "bd71828-rtc", + .resources = rtc_irqs, + .num_resources = ARRAY_SIZE(rtc_irqs), + }, { + .name = "gpio-keys", + .platform_data = &bd71828_powerkey_data, + .pdata_size = sizeof(bd71828_powerkey_data), + }, +}; + +static const struct regmap_range volatile_ranges[] = { + { + .range_min = BD71828_REG_PS_CTRL_1, + .range_max = BD71828_REG_PS_CTRL_1, + }, { + .range_min = BD71828_REG_PS_CTRL_3, + .range_max = BD71828_REG_PS_CTRL_3, + }, { + .range_min = BD71828_REG_RTC_SEC, + .range_max = BD71828_REG_RTC_YEAR, + }, { + /* + * For now make all charger registers volatile because many + * needs to be and because the charger block is not that + * performance critical. + */ + .range_min = BD71828_REG_CHG_STATE, + .range_max = BD71828_REG_CHG_FULL, + }, { + .range_min = BD71828_REG_INT_MAIN, + .range_max = BD71828_REG_IO_STAT, + }, +}; + +static const struct regmap_access_table volatile_regs = { + .yes_ranges = &volatile_ranges[0], + .n_yes_ranges = ARRAY_SIZE(volatile_ranges), +}; + +static struct regmap_config bd71828_regmap = { + .reg_bits = 8, + .val_bits = 8, + .volatile_table = &volatile_regs, + .max_register = BD71828_MAX_REGISTER, + .cache_type = REGCACHE_RBTREE, +}; + +/* + * Mapping of main IRQ register bits to sub-IRQ register offsets so that we can + * access corect sub-IRQ registers based on bits that are set in main IRQ + * register. + */ + +static unsigned int bit0_offsets[] = {11}; /* RTC IRQ */ +static unsigned int bit1_offsets[] = {10}; /* TEMP IRQ */ +static unsigned int bit2_offsets[] = {6, 7, 8, 9}; /* BAT MON IRQ */ +static unsigned int bit3_offsets[] = {5}; /* BAT IRQ */ +static unsigned int bit4_offsets[] = {4}; /* CHG IRQ */ +static unsigned int bit5_offsets[] = {3}; /* VSYS IRQ */ +static unsigned int bit6_offsets[] = {1, 2}; /* DCIN IRQ */ +static unsigned int bit7_offsets[] = {0}; /* BUCK IRQ */ + +static struct regmap_irq_sub_irq_map bd71828_sub_irq_offsets[] = { + REGMAP_IRQ_MAIN_REG_OFFSET(bit0_offsets), + REGMAP_IRQ_MAIN_REG_OFFSET(bit1_offsets), + REGMAP_IRQ_MAIN_REG_OFFSET(bit2_offsets), + REGMAP_IRQ_MAIN_REG_OFFSET(bit3_offsets), + REGMAP_IRQ_MAIN_REG_OFFSET(bit4_offsets), + REGMAP_IRQ_MAIN_REG_OFFSET(bit5_offsets), + REGMAP_IRQ_MAIN_REG_OFFSET(bit6_offsets), + REGMAP_IRQ_MAIN_REG_OFFSET(bit7_offsets), +}; + +static struct regmap_irq bd71828_irqs[] = { + REGMAP_IRQ_REG(BD71828_INT_BUCK1_OCP, 0, BD71828_INT_BUCK1_OCP_MASK), + REGMAP_IRQ_REG(BD71828_INT_BUCK2_OCP, 0, BD71828_INT_BUCK2_OCP_MASK), + REGMAP_IRQ_REG(BD71828_INT_BUCK3_OCP, 0, BD71828_INT_BUCK3_OCP_MASK), + REGMAP_IRQ_REG(BD71828_INT_BUCK4_OCP, 0, BD71828_INT_BUCK4_OCP_MASK), + REGMAP_IRQ_REG(BD71828_INT_BUCK5_OCP, 0, BD71828_INT_BUCK5_OCP_MASK), + REGMAP_IRQ_REG(BD71828_INT_BUCK6_OCP, 0, BD71828_INT_BUCK6_OCP_MASK), + REGMAP_IRQ_REG(BD71828_INT_BUCK7_OCP, 0, BD71828_INT_BUCK7_OCP_MASK), + REGMAP_IRQ_REG(BD71828_INT_PGFAULT, 0, BD71828_INT_PGFAULT_MASK), + /* DCIN1 interrupts */ + REGMAP_IRQ_REG(BD71828_INT_DCIN_DET, 1, BD71828_INT_DCIN_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_DCIN_RMV, 1, BD71828_INT_DCIN_RMV_MASK), + REGMAP_IRQ_REG(BD71828_INT_CLPS_OUT, 1, BD71828_INT_CLPS_OUT_MASK), + REGMAP_IRQ_REG(BD71828_INT_CLPS_IN, 1, BD71828_INT_CLPS_IN_MASK), + /* DCIN2 interrupts */ + REGMAP_IRQ_REG(BD71828_INT_DCIN_MON_RES, 2, + BD71828_INT_DCIN_MON_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_DCIN_MON_DET, 2, + BD71828_INT_DCIN_MON_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_LONGPUSH, 2, BD71828_INT_LONGPUSH_MASK), + REGMAP_IRQ_REG(BD71828_INT_MIDPUSH, 2, BD71828_INT_MIDPUSH_MASK), + REGMAP_IRQ_REG(BD71828_INT_SHORTPUSH, 2, BD71828_INT_SHORTPUSH_MASK), + REGMAP_IRQ_REG(BD71828_INT_PUSH, 2, BD71828_INT_PUSH_MASK), + REGMAP_IRQ_REG(BD71828_INT_WDOG, 2, BD71828_INT_WDOG_MASK), + REGMAP_IRQ_REG(BD71828_INT_SWRESET, 2, BD71828_INT_SWRESET_MASK), + /* Vsys */ + REGMAP_IRQ_REG(BD71828_INT_VSYS_UV_RES, 3, + BD71828_INT_VSYS_UV_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_VSYS_UV_DET, 3, + BD71828_INT_VSYS_UV_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_VSYS_LOW_RES, 3, + BD71828_INT_VSYS_LOW_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_VSYS_LOW_DET, 3, + BD71828_INT_VSYS_LOW_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_VSYS_HALL_IN, 3, + BD71828_INT_VSYS_HALL_IN_MASK), + REGMAP_IRQ_REG(BD71828_INT_VSYS_HALL_TOGGLE, 3, + BD71828_INT_VSYS_HALL_TOGGLE_MASK), + REGMAP_IRQ_REG(BD71828_INT_VSYS_MON_RES, 3, + BD71828_INT_VSYS_MON_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_VSYS_MON_DET, 3, + BD71828_INT_VSYS_MON_DET_MASK), + /* Charger */ + REGMAP_IRQ_REG(BD71828_INT_CHG_DCIN_ILIM, 4, + BD71828_INT_CHG_DCIN_ILIM_MASK), + REGMAP_IRQ_REG(BD71828_INT_CHG_TOPOFF_TO_DONE, 4, + BD71828_INT_CHG_TOPOFF_TO_DONE_MASK), + REGMAP_IRQ_REG(BD71828_INT_CHG_WDG_TEMP, 4, + BD71828_INT_CHG_WDG_TEMP_MASK), + REGMAP_IRQ_REG(BD71828_INT_CHG_WDG_TIME, 4, + BD71828_INT_CHG_WDG_TIME_MASK), + REGMAP_IRQ_REG(BD71828_INT_CHG_RECHARGE_RES, 4, + BD71828_INT_CHG_RECHARGE_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_CHG_RECHARGE_DET, 4, + BD71828_INT_CHG_RECHARGE_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_CHG_RANGED_TEMP_TRANSITION, 4, + BD71828_INT_CHG_RANGED_TEMP_TRANSITION_MASK), + REGMAP_IRQ_REG(BD71828_INT_CHG_STATE_TRANSITION, 4, + BD71828_INT_CHG_STATE_TRANSITION_MASK), + /* Battery */ + REGMAP_IRQ_REG(BD71828_INT_BAT_TEMP_NORMAL, 5, + BD71828_INT_BAT_TEMP_NORMAL_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_TEMP_ERANGE, 5, + BD71828_INT_BAT_TEMP_ERANGE_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_TEMP_WARN, 5, + BD71828_INT_BAT_TEMP_WARN_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_REMOVED, 5, + BD71828_INT_BAT_REMOVED_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_DETECTED, 5, + BD71828_INT_BAT_DETECTED_MASK), + REGMAP_IRQ_REG(BD71828_INT_THERM_REMOVED, 5, + BD71828_INT_THERM_REMOVED_MASK), + REGMAP_IRQ_REG(BD71828_INT_THERM_DETECTED, 5, + BD71828_INT_THERM_DETECTED_MASK), + /* Battery Mon 1 */ + REGMAP_IRQ_REG(BD71828_INT_BAT_DEAD, 6, BD71828_INT_BAT_DEAD_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_SHORTC_RES, 6, + BD71828_INT_BAT_SHORTC_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_SHORTC_DET, 6, + BD71828_INT_BAT_SHORTC_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_LOW_VOLT_RES, 6, + BD71828_INT_BAT_LOW_VOLT_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_LOW_VOLT_DET, 6, + BD71828_INT_BAT_LOW_VOLT_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_VOLT_RES, 6, + BD71828_INT_BAT_OVER_VOLT_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_VOLT_DET, 6, + BD71828_INT_BAT_OVER_VOLT_DET_MASK), + /* Battery Mon 2 */ + REGMAP_IRQ_REG(BD71828_INT_BAT_MON_RES, 7, + BD71828_INT_BAT_MON_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_MON_DET, 7, + BD71828_INT_BAT_MON_DET_MASK), + /* Battery Mon 3 (Coulomb counter) */ + REGMAP_IRQ_REG(BD71828_INT_BAT_CC_MON1, 8, + BD71828_INT_BAT_CC_MON1_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_CC_MON2, 8, + BD71828_INT_BAT_CC_MON2_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_CC_MON3, 8, + BD71828_INT_BAT_CC_MON3_MASK), + /* Battery Mon 4 */ + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_CURR_1_RES, 9, + BD71828_INT_BAT_OVER_CURR_1_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_CURR_1_DET, 9, + BD71828_INT_BAT_OVER_CURR_1_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_CURR_2_RES, 9, + BD71828_INT_BAT_OVER_CURR_2_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_CURR_2_DET, 9, + BD71828_INT_BAT_OVER_CURR_2_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_CURR_3_RES, 9, + BD71828_INT_BAT_OVER_CURR_3_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_BAT_OVER_CURR_3_DET, 9, + BD71828_INT_BAT_OVER_CURR_3_DET_MASK), + /* Temperature */ + REGMAP_IRQ_REG(BD71828_INT_TEMP_BAT_LOW_RES, 10, + BD71828_INT_TEMP_BAT_LOW_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_TEMP_BAT_LOW_DET, 10, + BD71828_INT_TEMP_BAT_LOW_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_TEMP_BAT_HI_RES, 10, + BD71828_INT_TEMP_BAT_HI_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_TEMP_BAT_HI_DET, 10, + BD71828_INT_TEMP_BAT_HI_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_TEMP_CHIP_OVER_125_RES, 10, + BD71828_INT_TEMP_CHIP_OVER_125_RES_MASK), + REGMAP_IRQ_REG(BD71828_INT_TEMP_CHIP_OVER_125_DET, 10, + BD71828_INT_TEMP_CHIP_OVER_125_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_TEMP_CHIP_OVER_VF_DET, 10, + BD71828_INT_TEMP_CHIP_OVER_VF_DET_MASK), + REGMAP_IRQ_REG(BD71828_INT_TEMP_CHIP_OVER_VF_RES, 10, + BD71828_INT_TEMP_CHIP_OVER_VF_RES_MASK), + /* RTC Alarm */ + REGMAP_IRQ_REG(BD71828_INT_RTC0, 11, BD71828_INT_RTC0_MASK), + REGMAP_IRQ_REG(BD71828_INT_RTC1, 11, BD71828_INT_RTC1_MASK), + REGMAP_IRQ_REG(BD71828_INT_RTC2, 11, BD71828_INT_RTC2_MASK), +}; + +static struct regmap_irq_chip bd71828_irq_chip = { + .name = "bd71828_irq", + .main_status = BD71828_REG_INT_MAIN, + .irqs = &bd71828_irqs[0], + .num_irqs = ARRAY_SIZE(bd71828_irqs), + .status_base = BD71828_REG_INT_BUCK, + .mask_base = BD71828_REG_INT_MASK_BUCK, + .ack_base = BD71828_REG_INT_BUCK, + .mask_invert = true, + .init_ack_masked = true, + .num_regs = 12, + .num_main_regs = 1, + .sub_reg_offsets = &bd71828_sub_irq_offsets[0], + .num_main_status_bits = 8, + .irq_reg_stride = 1, +}; + +static int bd71828_i2c_probe(struct i2c_client *i2c) +{ + struct rohm_regmap_dev *chip; + struct regmap_irq_chip_data *irq_data; + int ret; + + if (!i2c->irq) { + dev_err(&i2c->dev, "No IRQ configured\n"); + return -EINVAL; + } + + chip = devm_kzalloc(&i2c->dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + dev_set_drvdata(&i2c->dev, chip); + + chip->regmap = devm_regmap_init_i2c(i2c, &bd71828_regmap); + if (IS_ERR(chip->regmap)) { + dev_err(&i2c->dev, "Failed to initialize Regmap\n"); + return PTR_ERR(chip->regmap); + } + + ret = devm_regmap_add_irq_chip(&i2c->dev, chip->regmap, + i2c->irq, IRQF_ONESHOT, 0, + &bd71828_irq_chip, &irq_data); + if (ret) { + dev_err(&i2c->dev, "Failed to add IRQ chip\n"); + return ret; + } + + dev_dbg(&i2c->dev, "Registered %d IRQs for chip\n", + bd71828_irq_chip.num_irqs); + + ret = regmap_irq_get_virq(irq_data, BD71828_INT_SHORTPUSH); + if (ret < 0) { + dev_err(&i2c->dev, "Failed to get the power-key IRQ\n"); + return ret; + } + + button.irq = ret; + + ret = devm_mfd_add_devices(&i2c->dev, PLATFORM_DEVID_AUTO, + bd71828_mfd_cells, + ARRAY_SIZE(bd71828_mfd_cells), NULL, 0, + regmap_irq_get_domain(irq_data)); + if (ret) + dev_err(&i2c->dev, "Failed to create subdevices\n"); + + return ret; +} + +static const struct of_device_id bd71828_of_match[] = { + { .compatible = "rohm,bd71828", }, + { }, +}; +MODULE_DEVICE_TABLE(of, bd71828_of_match); + +static struct i2c_driver bd71828_drv = { + .driver = { + .name = "rohm-bd71828", + .of_match_table = bd71828_of_match, + }, + .probe_new = &bd71828_i2c_probe, +}; +module_i2c_driver(bd71828_drv); + +MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>"); +MODULE_DESCRIPTION("ROHM BD71828 Power Management IC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/mfd/rohm-bd718x7.c b/drivers/mfd/rohm-bd718x7.c index 85e7f5133365..c32c1b6c98fa 100644 --- a/drivers/mfd/rohm-bd718x7.c +++ b/drivers/mfd/rohm-bd718x7.c @@ -30,14 +30,24 @@ static struct gpio_keys_platform_data bd718xx_powerkey_data = { .name = "bd718xx-pwrkey", }; -static struct mfd_cell bd718xx_mfd_cells[] = { +static struct mfd_cell bd71837_mfd_cells[] = { { .name = "gpio-keys", .platform_data = &bd718xx_powerkey_data, .pdata_size = sizeof(bd718xx_powerkey_data), }, - { .name = "bd718xx-clk", }, - { .name = "bd718xx-pmic", }, + { .name = "bd71837-clk", }, + { .name = "bd71837-pmic", }, +}; + +static struct mfd_cell bd71847_mfd_cells[] = { + { + .name = "gpio-keys", + .platform_data = &bd718xx_powerkey_data, + .pdata_size = sizeof(bd718xx_powerkey_data), + }, + { .name = "bd71847-clk", }, + { .name = "bd71847-pmic", }, }; static const struct regmap_irq bd718xx_irqs[] = { @@ -124,6 +134,9 @@ static int bd718xx_i2c_probe(struct i2c_client *i2c, { struct bd718xx *bd718xx; int ret; + unsigned int chip_type; + struct mfd_cell *mfd; + int cells; if (!i2c->irq) { dev_err(&i2c->dev, "No IRQ configured\n"); @@ -136,8 +149,21 @@ static int bd718xx_i2c_probe(struct i2c_client *i2c, return -ENOMEM; bd718xx->chip_irq = i2c->irq; - bd718xx->chip.chip_type = (unsigned int)(uintptr_t) - of_device_get_match_data(&i2c->dev); + chip_type = (unsigned int)(uintptr_t) + of_device_get_match_data(&i2c->dev); + switch (chip_type) { + case ROHM_CHIP_TYPE_BD71837: + mfd = bd71837_mfd_cells; + cells = ARRAY_SIZE(bd71837_mfd_cells); + break; + case ROHM_CHIP_TYPE_BD71847: + mfd = bd71847_mfd_cells; + cells = ARRAY_SIZE(bd71847_mfd_cells); + break; + default: + dev_err(&i2c->dev, "Unknown device type"); + return -EINVAL; + } bd718xx->chip.dev = &i2c->dev; dev_set_drvdata(&i2c->dev, bd718xx); @@ -170,8 +196,7 @@ static int bd718xx_i2c_probe(struct i2c_client *i2c, button.irq = ret; ret = devm_mfd_add_devices(bd718xx->chip.dev, PLATFORM_DEVID_AUTO, - bd718xx_mfd_cells, - ARRAY_SIZE(bd718xx_mfd_cells), NULL, 0, + mfd, cells, NULL, 0, regmap_irq_get_domain(bd718xx->irq_data)); if (ret) dev_err(&i2c->dev, "Failed to create subdevices\n"); @@ -188,6 +213,10 @@ static const struct of_device_id bd718xx_of_match[] = { .compatible = "rohm,bd71847", .data = (void *)ROHM_CHIP_TYPE_BD71847, }, + { + .compatible = "rohm,bd71850", + .data = (void *)ROHM_CHIP_TYPE_BD71847, + }, { } }; MODULE_DEVICE_TABLE(of, bd718xx_of_match); diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c index 154270f8d8d7..e49787e6bb93 100644 --- a/drivers/mfd/sm501.c +++ b/drivers/mfd/sm501.c @@ -1086,8 +1086,7 @@ static int sm501_register_gpio(struct sm501_devdata *sm) iounmap(gpio->regs); err_claimed: - release_resource(gpio->regs_res); - kfree(gpio->regs_res); + release_mem_region(iobase, 0x20); return ret; } @@ -1095,6 +1094,7 @@ static int sm501_register_gpio(struct sm501_devdata *sm) static void sm501_gpio_remove(struct sm501_devdata *sm) { struct sm501_gpio *gpio = &sm->gpio; + resource_size_t iobase = sm->io_res->start + SM501_GPIO; if (!sm->gpio.registered) return; @@ -1103,8 +1103,7 @@ static void sm501_gpio_remove(struct sm501_devdata *sm) gpiochip_remove(&gpio->high.gpio); iounmap(gpio->regs); - release_resource(gpio->regs_res); - kfree(gpio->regs_res); + release_mem_region(iobase, 0x20); } static inline int sm501_gpio_isregistered(struct sm501_devdata *sm) @@ -1427,8 +1426,7 @@ static int sm501_plat_probe(struct platform_device *dev) return sm501_init_dev(sm); err_claim: - release_resource(sm->regs_claim); - kfree(sm->regs_claim); + release_mem_region(sm->io_res->start, 0x100); err_res: kfree(sm); err1: @@ -1637,8 +1635,7 @@ static int sm501_pci_probe(struct pci_dev *dev, return 0; err4: - release_resource(sm->regs_claim); - kfree(sm->regs_claim); + release_mem_region(sm->io_res->start, 0x100); err3: pci_disable_device(dev); err2: @@ -1673,8 +1670,7 @@ static void sm501_pci_remove(struct pci_dev *dev) sm501_dev_remove(sm); iounmap(sm->regs); - release_resource(sm->regs_claim); - kfree(sm->regs_claim); + release_mem_region(sm->io_res->start, 0x100); pci_disable_device(dev); } @@ -1686,8 +1682,7 @@ static int sm501_plat_remove(struct platform_device *dev) sm501_dev_remove(sm); iounmap(sm->regs); - release_resource(sm->regs_claim); - kfree(sm->regs_claim); + release_mem_region(sm->io_res->start, 0x100); return 0; } diff --git a/drivers/mfd/syscon.c b/drivers/mfd/syscon.c index e22197c832e8..3a97816d0cba 100644 --- a/drivers/mfd/syscon.c +++ b/drivers/mfd/syscon.c @@ -224,6 +224,35 @@ struct regmap *syscon_regmap_lookup_by_phandle(struct device_node *np, } EXPORT_SYMBOL_GPL(syscon_regmap_lookup_by_phandle); +struct regmap *syscon_regmap_lookup_by_phandle_args(struct device_node *np, + const char *property, + int arg_count, + unsigned int *out_args) +{ + struct device_node *syscon_np; + struct of_phandle_args args; + struct regmap *regmap; + unsigned int index; + int rc; + + rc = of_parse_phandle_with_fixed_args(np, property, arg_count, + 0, &args); + if (rc) + return ERR_PTR(rc); + + syscon_np = args.np; + if (!syscon_np) + return ERR_PTR(-ENODEV); + + regmap = syscon_node_to_regmap(syscon_np); + for (index = 0; index < arg_count; index++) + out_args[index] = args.args[index]; + of_node_put(syscon_np); + + return regmap; +} +EXPORT_SYMBOL_GPL(syscon_regmap_lookup_by_phandle_args); + static int syscon_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -245,7 +274,7 @@ static int syscon_probe(struct platform_device *pdev) if (!base) return -ENOMEM; - syscon_config.max_register = res->end - res->start - 3; + syscon_config.max_register = resource_size(res) - 4; if (pdata) syscon_config.name = pdata->label; syscon->regmap = devm_regmap_init_mmio(dev, base, &syscon_config); diff --git a/drivers/mfd/tqmx86.c b/drivers/mfd/tqmx86.c index 22d2f02d855c..b9f48e588d95 100644 --- a/drivers/mfd/tqmx86.c +++ b/drivers/mfd/tqmx86.c @@ -158,7 +158,7 @@ static int tqmx86_board_id_to_clk_rate(u8 board_id) static int tqmx86_probe(struct platform_device *pdev) { - u8 board_id, rev, i2c_det, i2c_ien, io_ext_int_val; + u8 board_id, rev, i2c_det, io_ext_int_val; struct device *dev = &pdev->dev; u8 gpio_irq_cfg, readback; const char *board_name; @@ -196,7 +196,6 @@ static int tqmx86_probe(struct platform_device *pdev) board_name, board_id, rev >> 4, rev & 0xf); i2c_det = ioread8(io_base + TQMX86_REG_I2C_DETECT); - i2c_ien = ioread8(io_base + TQMX86_REG_I2C_INT_EN); if (gpio_irq_cfg) { io_ext_int_val = diff --git a/drivers/mfd/wcd934x.c b/drivers/mfd/wcd934x.c new file mode 100644 index 000000000000..90341f3c6810 --- /dev/null +++ b/drivers/mfd/wcd934x.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2019, Linaro Limited + +#include <linux/clk.h> +#include <linux/gpio.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/mfd/core.h> +#include <linux/mfd/wcd934x/registers.h> +#include <linux/mfd/wcd934x/wcd934x.h> +#include <linux/module.h> +#include <linux/of_gpio.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/regulator/consumer.h> +#include <linux/slimbus.h> + +static const struct mfd_cell wcd934x_devices[] = { + { + .name = "wcd934x-codec", + }, { + .name = "wcd934x-gpio", + .of_compatible = "qcom,wcd9340-gpio", + }, { + .name = "wcd934x-soundwire", + .of_compatible = "qcom,soundwire-v1.3.0", + }, +}; + +static const struct regmap_irq wcd934x_irqs[] = { + [WCD934X_IRQ_SLIMBUS] = { + .reg_offset = 0, + .mask = BIT(0), + .type = { + .type_reg_offset = 0, + .types_supported = IRQ_TYPE_EDGE_BOTH, + .type_reg_mask = BIT(0), + .type_level_low_val = BIT(0), + .type_level_high_val = BIT(0), + .type_falling_val = 0, + .type_rising_val = 0, + }, + }, + [WCD934X_IRQ_SOUNDWIRE] = { + .reg_offset = 2, + .mask = BIT(4), + .type = { + .type_reg_offset = 2, + .types_supported = IRQ_TYPE_EDGE_BOTH, + .type_reg_mask = BIT(4), + .type_level_low_val = BIT(4), + .type_level_high_val = BIT(4), + .type_falling_val = 0, + .type_rising_val = 0, + }, + }, +}; + +static const struct regmap_irq_chip wcd934x_regmap_irq_chip = { + .name = "wcd934x_irq", + .status_base = WCD934X_INTR_PIN1_STATUS0, + .mask_base = WCD934X_INTR_PIN1_MASK0, + .ack_base = WCD934X_INTR_PIN1_CLEAR0, + .type_base = WCD934X_INTR_LEVEL0, + .num_type_reg = 4, + .type_in_mask = false, + .num_regs = 4, + .irqs = wcd934x_irqs, + .num_irqs = ARRAY_SIZE(wcd934x_irqs), +}; + +static bool wcd934x_is_volatile_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case WCD934X_INTR_PIN1_STATUS0...WCD934X_INTR_PIN2_CLEAR3: + case WCD934X_SWR_AHB_BRIDGE_RD_DATA_0: + case WCD934X_SWR_AHB_BRIDGE_RD_DATA_1: + case WCD934X_SWR_AHB_BRIDGE_RD_DATA_2: + case WCD934X_SWR_AHB_BRIDGE_RD_DATA_3: + case WCD934X_SWR_AHB_BRIDGE_ACCESS_STATUS: + case WCD934X_ANA_MBHC_RESULT_3: + case WCD934X_ANA_MBHC_RESULT_2: + case WCD934X_ANA_MBHC_RESULT_1: + case WCD934X_ANA_MBHC_MECH: + case WCD934X_ANA_MBHC_ELECT: + case WCD934X_ANA_MBHC_ZDET: + case WCD934X_ANA_MICB2: + case WCD934X_ANA_RCO: + case WCD934X_ANA_BIAS: + return true; + default: + return false; + } +}; + +static const struct regmap_range_cfg wcd934x_ranges[] = { + { .name = "WCD934X", + .range_min = 0x0, + .range_max = WCD934X_MAX_REGISTER, + .selector_reg = WCD934X_SEL_REGISTER, + .selector_mask = WCD934X_SEL_MASK, + .selector_shift = WCD934X_SEL_SHIFT, + .window_start = WCD934X_WINDOW_START, + .window_len = WCD934X_WINDOW_LENGTH, + }, +}; + +static struct regmap_config wcd934x_regmap_config = { + .reg_bits = 16, + .val_bits = 8, + .cache_type = REGCACHE_RBTREE, + .max_register = 0xffff, + .can_multi_write = true, + .ranges = wcd934x_ranges, + .num_ranges = ARRAY_SIZE(wcd934x_ranges), + .volatile_reg = wcd934x_is_volatile_register, +}; + +static int wcd934x_bring_up(struct wcd934x_ddata *ddata) +{ + struct regmap *regmap = ddata->regmap; + u16 id_minor, id_major; + int ret; + + ret = regmap_bulk_read(regmap, WCD934X_CHIP_TIER_CTRL_CHIP_ID_BYTE0, + (u8 *)&id_minor, sizeof(u16)); + if (ret) + return ret; + + ret = regmap_bulk_read(regmap, WCD934X_CHIP_TIER_CTRL_CHIP_ID_BYTE2, + (u8 *)&id_major, sizeof(u16)); + if (ret) + return ret; + + dev_info(ddata->dev, "WCD934x chip id major 0x%x, minor 0x%x\n", + id_major, id_minor); + + regmap_write(regmap, WCD934X_CODEC_RPM_RST_CTL, 0x01); + regmap_write(regmap, WCD934X_SIDO_NEW_VOUT_A_STARTUP, 0x19); + regmap_write(regmap, WCD934X_SIDO_NEW_VOUT_D_STARTUP, 0x15); + /* Add 1msec delay for VOUT to settle */ + usleep_range(1000, 1100); + regmap_write(regmap, WCD934X_CODEC_RPM_PWR_CDC_DIG_HM_CTL, 0x5); + regmap_write(regmap, WCD934X_CODEC_RPM_PWR_CDC_DIG_HM_CTL, 0x7); + regmap_write(regmap, WCD934X_CODEC_RPM_RST_CTL, 0x3); + regmap_write(regmap, WCD934X_CODEC_RPM_RST_CTL, 0x7); + regmap_write(regmap, WCD934X_CODEC_RPM_PWR_CDC_DIG_HM_CTL, 0x3); + + return 0; +} + +static int wcd934x_slim_status_up(struct slim_device *sdev) +{ + struct device *dev = &sdev->dev; + struct wcd934x_ddata *ddata; + int ret; + + ddata = dev_get_drvdata(dev); + + ddata->regmap = regmap_init_slimbus(sdev, &wcd934x_regmap_config); + if (IS_ERR(ddata->regmap)) { + dev_err(dev, "Error allocating slim regmap\n"); + return PTR_ERR(ddata->regmap); + } + + ret = wcd934x_bring_up(ddata); + if (ret) { + dev_err(dev, "Failed to bring up WCD934X: err = %d\n", ret); + return ret; + } + + ret = devm_regmap_add_irq_chip(dev, ddata->regmap, ddata->irq, + IRQF_TRIGGER_HIGH, 0, + &wcd934x_regmap_irq_chip, + &ddata->irq_data); + if (ret) { + dev_err(dev, "Failed to add IRQ chip: err = %d\n", ret); + return ret; + } + + ret = mfd_add_devices(dev, PLATFORM_DEVID_AUTO, wcd934x_devices, + ARRAY_SIZE(wcd934x_devices), NULL, 0, NULL); + if (ret) { + dev_err(dev, "Failed to add child devices: err = %d\n", + ret); + return ret; + } + + return ret; +} + +static int wcd934x_slim_status(struct slim_device *sdev, + enum slim_device_status status) +{ + switch (status) { + case SLIM_DEVICE_STATUS_UP: + return wcd934x_slim_status_up(sdev); + case SLIM_DEVICE_STATUS_DOWN: + mfd_remove_devices(&sdev->dev); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int wcd934x_slim_probe(struct slim_device *sdev) +{ + struct device *dev = &sdev->dev; + struct device_node *np = dev->of_node; + struct wcd934x_ddata *ddata; + int reset_gpio, ret; + + ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL); + if (!ddata) + return -ENOMEM; + + ddata->irq = of_irq_get(np, 0); + if (ddata->irq < 0) { + if (ddata->irq != -EPROBE_DEFER) + dev_err(ddata->dev, "Failed to get IRQ: err = %d\n", + ddata->irq); + return ddata->irq; + } + + reset_gpio = of_get_named_gpio(np, "reset-gpios", 0); + if (reset_gpio < 0) { + dev_err(dev, "Failed to get reset gpio: err = %d\n", + reset_gpio); + return reset_gpio; + } + + ddata->extclk = devm_clk_get(dev, "extclk"); + if (IS_ERR(ddata->extclk)) { + dev_err(dev, "Failed to get extclk"); + return PTR_ERR(ddata->extclk); + } + + ddata->supplies[0].supply = "vdd-buck"; + ddata->supplies[1].supply = "vdd-buck-sido"; + ddata->supplies[2].supply = "vdd-tx"; + ddata->supplies[3].supply = "vdd-rx"; + ddata->supplies[4].supply = "vdd-io"; + + ret = regulator_bulk_get(dev, WCD934X_MAX_SUPPLY, ddata->supplies); + if (ret) { + dev_err(dev, "Failed to get supplies: err = %d\n", ret); + return ret; + } + + ret = regulator_bulk_enable(WCD934X_MAX_SUPPLY, ddata->supplies); + if (ret) { + dev_err(dev, "Failed to enable supplies: err = %d\n", ret); + return ret; + } + + /* + * For WCD934X, it takes about 600us for the Vout_A and + * Vout_D to be ready after BUCK_SIDO is powered up. + * SYS_RST_N shouldn't be pulled high during this time + */ + usleep_range(600, 650); + gpio_direction_output(reset_gpio, 0); + msleep(20); + gpio_set_value(reset_gpio, 1); + msleep(20); + + ddata->dev = dev; + dev_set_drvdata(dev, ddata); + + return 0; +} + +static void wcd934x_slim_remove(struct slim_device *sdev) +{ + struct wcd934x_ddata *ddata = dev_get_drvdata(&sdev->dev); + + regulator_bulk_disable(WCD934X_MAX_SUPPLY, ddata->supplies); + mfd_remove_devices(&sdev->dev); + kfree(ddata); +} + +static const struct slim_device_id wcd934x_slim_id[] = { + { SLIM_MANF_ID_QCOM, SLIM_PROD_CODE_WCD9340, + SLIM_DEV_IDX_WCD9340, SLIM_DEV_INSTANCE_ID_WCD9340 }, + {} +}; + +static struct slim_driver wcd934x_slim_driver = { + .driver = { + .name = "wcd934x-slim", + }, + .probe = wcd934x_slim_probe, + .remove = wcd934x_slim_remove, + .device_status = wcd934x_slim_status, + .id_table = wcd934x_slim_id, +}; + +module_slim_driver(wcd934x_slim_driver); +MODULE_DESCRIPTION("WCD934X slim driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("slim:217:250:*"); +MODULE_AUTHOR("Srinivas Kandagatla <srinivas.kandagatla@linaro.org>"); diff --git a/drivers/misc/ocxl/Kconfig b/drivers/misc/ocxl/Kconfig index 1916fa65f2f2..2d2266c1439e 100644 --- a/drivers/misc/ocxl/Kconfig +++ b/drivers/misc/ocxl/Kconfig @@ -11,6 +11,7 @@ config OCXL tristate "OpenCAPI coherent accelerator support" depends on PPC_POWERNV && PCI && EEH select OCXL_BASE + select HOTPLUG_PCI_POWERNV default m help Select this option to enable the ocxl driver for Open diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 2817f4751306..97b8b38ab47d 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -255,28 +255,28 @@ static int options_open(struct inode *inode, struct file *file) } /* *INDENT-OFF* */ -static const struct file_operations statistics_fops = { - .open = statistics_open, - .read = seq_read, - .write = statistics_write, - .llseek = seq_lseek, - .release = single_release, +static const struct proc_ops statistics_proc_ops = { + .proc_open = statistics_open, + .proc_read = seq_read, + .proc_write = statistics_write, + .proc_lseek = seq_lseek, + .proc_release = single_release, }; -static const struct file_operations mcs_statistics_fops = { - .open = mcs_statistics_open, - .read = seq_read, - .write = mcs_statistics_write, - .llseek = seq_lseek, - .release = single_release, +static const struct proc_ops mcs_statistics_proc_ops = { + .proc_open = mcs_statistics_open, + .proc_read = seq_read, + .proc_write = mcs_statistics_write, + .proc_lseek = seq_lseek, + .proc_release = single_release, }; -static const struct file_operations options_fops = { - .open = options_open, - .read = seq_read, - .write = options_write, - .llseek = seq_lseek, - .release = single_release, +static const struct proc_ops options_proc_ops = { + .proc_open = options_open, + .proc_read = seq_read, + .proc_write = options_write, + .proc_lseek = seq_lseek, + .proc_release = single_release, }; static struct proc_dir_entry *proc_gru __read_mostly; @@ -286,11 +286,11 @@ int gru_proc_init(void) proc_gru = proc_mkdir("sgi_uv/gru", NULL); if (!proc_gru) return -1; - if (!proc_create("statistics", 0644, proc_gru, &statistics_fops)) + if (!proc_create("statistics", 0644, proc_gru, &statistics_proc_ops)) goto err; - if (!proc_create("mcs_statistics", 0644, proc_gru, &mcs_statistics_fops)) + if (!proc_create("mcs_statistics", 0644, proc_gru, &mcs_statistics_proc_ops)) goto err; - if (!proc_create("debug_options", 0644, proc_gru, &options_fops)) + if (!proc_create("debug_options", 0644, proc_gru, &options_proc_ops)) goto err; if (!proc_create_seq("cch_status", 0444, proc_gru, &cch_seq_ops)) goto err; diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index 25fb72b2efa0..2f93c25bbaee 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -1180,7 +1180,7 @@ static struct mtd_info * __init open_mtd_device(const char *mtd_dev) * MTD device name. */ mtd = get_mtd_device_nm(mtd_dev); - if (IS_ERR(mtd) && PTR_ERR(mtd) == -ENODEV) + if (PTR_ERR(mtd) == -ENODEV) /* Probably this is an MTD character device node path */ mtd = open_mtd_by_chdev(mtd_dev); } else diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 483935b001c8..597e6fd5bfea 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -7893,7 +7893,7 @@ static void bnxt_setup_msix(struct bnxt *bp) int tcs, i; tcs = netdev_get_num_tc(dev); - if (tcs > 1) { + if (tcs) { int i, off, count; for (i = 0; i < tcs; i++) { @@ -9241,6 +9241,17 @@ void bnxt_half_close_nic(struct bnxt *bp) bnxt_free_mem(bp, false); } +static void bnxt_reenable_sriov(struct bnxt *bp) +{ + if (BNXT_PF(bp)) { + struct bnxt_pf_info *pf = &bp->pf; + int n = pf->active_vfs; + + if (n) + bnxt_cfg_hw_sriov(bp, &n, true); + } +} + static int bnxt_open(struct net_device *dev) { struct bnxt *bp = netdev_priv(dev); @@ -9259,15 +9270,10 @@ static int bnxt_open(struct net_device *dev) bnxt_hwrm_if_change(bp, false); } else { if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - int n = pf->active_vfs; - - if (n) - bnxt_cfg_hw_sriov(bp, &n, true); - } - if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) + if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { bnxt_ulp_start(bp, 0); + bnxt_reenable_sriov(bp); + } } bnxt_hwmon_open(bp); } @@ -9307,10 +9313,6 @@ static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bnxt_debug_dev_exit(bp); bnxt_disable_napi(bp); del_timer_sync(&bp->timer); - if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && - pci_is_enabled(bp->pdev)) - pci_disable_device(bp->pdev); - bnxt_free_skbs(bp); /* Save ring stats before shutdown */ @@ -10096,9 +10098,16 @@ static void bnxt_reset(struct bnxt *bp, bool silent) static void bnxt_fw_reset_close(struct bnxt *bp) { bnxt_ulp_stop(bp); + /* When firmware is fatal state, disable PCI device to prevent + * any potential bad DMAs before freeing kernel memory. + */ + if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) + pci_disable_device(bp->pdev); __bnxt_close_nic(bp, true, false); bnxt_clear_int_mode(bp); bnxt_hwrm_func_drv_unrgtr(bp); + if (pci_is_enabled(bp->pdev)) + pci_disable_device(bp->pdev); bnxt_free_ctx_mem(bp); kfree(bp->ctx); bp->ctx = NULL; @@ -10831,6 +10840,8 @@ static void bnxt_fw_reset_task(struct work_struct *work) smp_mb__before_atomic(); clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); bnxt_ulp_start(bp, rc); + if (!rc) + bnxt_reenable_sriov(bp); bnxt_dl_health_recovery_done(bp); bnxt_dl_health_status_update(bp, true); rtnl_unlock(); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 8247d21d0432..b945bd3d5d88 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -171,9 +171,9 @@ static int otx2_hw_get_mac_addr(struct otx2_nic *pfvf, } msghdr = otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); - if (!msghdr) { + if (IS_ERR(msghdr)) { otx2_mbox_unlock(&pfvf->mbox); - return -ENOMEM; + return PTR_ERR(msghdr); } rsp = (struct nix_get_mac_addr_rsp *)msghdr; ether_addr_copy(netdev->dev_addr, rsp->mac_addr); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c index 79a2801d59f6..02526c53d4f5 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c @@ -614,7 +614,7 @@ mlxsw_sp_qdisc_tbf_rate_kbps(struct tc_tbf_qopt_offload_replace_params *p) /* TBF interface is in bytes/s, whereas Spectrum ASIC is configured in * Kbits/s. */ - return p->rate.rate_bytes_ps / 1000 * 8; + return div_u64(p->rate.rate_bytes_ps, 1000) * 8; } static int diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h index f131adad96e3..ce07c2931a72 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_if.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h @@ -866,7 +866,7 @@ struct ionic_rxq_comp { #define IONIC_RXQ_COMP_CSUM_F_VLAN 0x40 #define IONIC_RXQ_COMP_CSUM_F_CALC 0x80 u8 pkt_type_color; -#define IONIC_RXQ_COMP_PKT_TYPE_MASK 0x0f +#define IONIC_RXQ_COMP_PKT_TYPE_MASK 0x7f }; enum ionic_pkt_type { diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c index fbfff2b1dc93..1a636bad717d 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c +++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c @@ -1398,14 +1398,11 @@ void qed_qm_init_pf(struct qed_hwfn *p_hwfn, { struct qed_qm_info *qm_info = &p_hwfn->qm_info; struct qed_qm_pf_rt_init_params params; - struct qed_mcp_link_state *p_link; struct qed_qm_iids iids; memset(&iids, 0, sizeof(iids)); qed_cxt_qm_iids(p_hwfn, &iids); - p_link = &QED_LEADING_HWFN(p_hwfn->cdev)->mcp_info->link_output; - memset(¶ms, 0, sizeof(params)); params.port_id = p_hwfn->port_id; params.pf_id = p_hwfn->rel_pf_id; diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c index 7912911337d4..03bdd2e26329 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_dev.c +++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c @@ -3114,6 +3114,7 @@ int qed_hw_init(struct qed_dev *cdev, struct qed_hw_init_params *p_params) if (!p_hwfn->fw_overlay_mem) { DP_NOTICE(p_hwfn, "Failed to allocate fw overlay memory\n"); + rc = -ENOMEM; goto load_err; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index ff1cbfc834b0..5836b21edd7e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -4974,6 +4974,7 @@ int stmmac_suspend(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct stmmac_priv *priv = netdev_priv(ndev); + u32 chan; if (!ndev || !netif_running(ndev)) return 0; @@ -4987,6 +4988,9 @@ int stmmac_suspend(struct device *dev) stmmac_disable_all_queues(priv); + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) + del_timer_sync(&priv->tx_queue[chan].txtimer); + /* Stop TX/RX DMA */ stmmac_stop_all_dma(priv); diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 7032a2405e1a..af07ea760b35 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -767,12 +767,12 @@ static int gtp_hashtable_new(struct gtp_dev *gtp, int hsize) int i; gtp->addr_hash = kmalloc_array(hsize, sizeof(struct hlist_head), - GFP_KERNEL); + GFP_KERNEL | __GFP_NOWARN); if (gtp->addr_hash == NULL) return -ENOMEM; gtp->tid_hash = kmalloc_array(hsize, sizeof(struct hlist_head), - GFP_KERNEL); + GFP_KERNEL | __GFP_NOWARN); if (gtp->tid_hash == NULL) goto err1; diff --git a/drivers/net/netdevsim/bpf.c b/drivers/net/netdevsim/bpf.c index 2b74425822ab..0b362b8dac17 100644 --- a/drivers/net/netdevsim/bpf.c +++ b/drivers/net/netdevsim/bpf.c @@ -218,6 +218,7 @@ static int nsim_bpf_create_prog(struct nsim_dev *nsim_dev, { struct nsim_bpf_bound_prog *state; char name[16]; + int ret; state = kzalloc(sizeof(*state), GFP_KERNEL); if (!state) @@ -230,9 +231,10 @@ static int nsim_bpf_create_prog(struct nsim_dev *nsim_dev, /* Program id is not populated yet when we create the state. */ sprintf(name, "%u", nsim_dev->prog_id_gen++); state->ddir = debugfs_create_dir(name, nsim_dev->ddir_bpf_bound_progs); - if (IS_ERR_OR_NULL(state->ddir)) { + if (IS_ERR(state->ddir)) { + ret = PTR_ERR(state->ddir); kfree(state); - return -ENOMEM; + return ret; } debugfs_create_u32("id", 0400, state->ddir, &prog->aux->id); @@ -587,8 +589,8 @@ int nsim_bpf_dev_init(struct nsim_dev *nsim_dev) nsim_dev->ddir_bpf_bound_progs = debugfs_create_dir("bpf_bound_progs", nsim_dev->ddir); - if (IS_ERR_OR_NULL(nsim_dev->ddir_bpf_bound_progs)) - return -ENOMEM; + if (IS_ERR(nsim_dev->ddir_bpf_bound_progs)) + return PTR_ERR(nsim_dev->ddir_bpf_bound_progs); nsim_dev->bpf_dev = bpf_offload_dev_create(&nsim_bpf_dev_ops, nsim_dev); err = PTR_ERR_OR_ZERO(nsim_dev->bpf_dev); diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c index 6aeed0c600f8..7971dc4f54f1 100644 --- a/drivers/net/netdevsim/bus.c +++ b/drivers/net/netdevsim/bus.c @@ -17,6 +17,7 @@ static DEFINE_IDA(nsim_bus_dev_ids); static LIST_HEAD(nsim_bus_dev_list); static DEFINE_MUTEX(nsim_bus_dev_list_lock); +static bool nsim_bus_enable; static struct nsim_bus_dev *to_nsim_bus_dev(struct device *dev) { @@ -28,7 +29,7 @@ static int nsim_bus_dev_vfs_enable(struct nsim_bus_dev *nsim_bus_dev, { nsim_bus_dev->vfconfigs = kcalloc(num_vfs, sizeof(struct nsim_vf_config), - GFP_KERNEL); + GFP_KERNEL | __GFP_NOWARN); if (!nsim_bus_dev->vfconfigs) return -ENOMEM; nsim_bus_dev->num_vfs = num_vfs; @@ -96,13 +97,25 @@ new_port_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev); + struct nsim_dev *nsim_dev = dev_get_drvdata(dev); + struct devlink *devlink; unsigned int port_index; int ret; + /* Prevent to use nsim_bus_dev before initialization. */ + if (!smp_load_acquire(&nsim_bus_dev->init)) + return -EBUSY; ret = kstrtouint(buf, 0, &port_index); if (ret) return ret; + + devlink = priv_to_devlink(nsim_dev); + + mutex_lock(&nsim_bus_dev->nsim_bus_reload_lock); + devlink_reload_disable(devlink); ret = nsim_dev_port_add(nsim_bus_dev, port_index); + devlink_reload_enable(devlink); + mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); return ret ? ret : count; } @@ -113,13 +126,25 @@ del_port_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev); + struct nsim_dev *nsim_dev = dev_get_drvdata(dev); + struct devlink *devlink; unsigned int port_index; int ret; + /* Prevent to use nsim_bus_dev before initialization. */ + if (!smp_load_acquire(&nsim_bus_dev->init)) + return -EBUSY; ret = kstrtouint(buf, 0, &port_index); if (ret) return ret; + + devlink = priv_to_devlink(nsim_dev); + + mutex_lock(&nsim_bus_dev->nsim_bus_reload_lock); + devlink_reload_disable(devlink); ret = nsim_dev_port_del(nsim_bus_dev, port_index); + devlink_reload_enable(devlink); + mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); return ret ? ret : count; } @@ -179,15 +204,30 @@ new_device_store(struct bus_type *bus, const char *buf, size_t count) pr_err("Format for adding new device is \"id port_count\" (uint uint).\n"); return -EINVAL; } - nsim_bus_dev = nsim_bus_dev_new(id, port_count); - if (IS_ERR(nsim_bus_dev)) - return PTR_ERR(nsim_bus_dev); mutex_lock(&nsim_bus_dev_list_lock); + /* Prevent to use resource before initialization. */ + if (!smp_load_acquire(&nsim_bus_enable)) { + err = -EBUSY; + goto err; + } + + nsim_bus_dev = nsim_bus_dev_new(id, port_count); + if (IS_ERR(nsim_bus_dev)) { + err = PTR_ERR(nsim_bus_dev); + goto err; + } + + /* Allow using nsim_bus_dev */ + smp_store_release(&nsim_bus_dev->init, true); + list_add_tail(&nsim_bus_dev->list, &nsim_bus_dev_list); mutex_unlock(&nsim_bus_dev_list_lock); return count; +err: + mutex_unlock(&nsim_bus_dev_list_lock); + return err; } static BUS_ATTR_WO(new_device); @@ -215,6 +255,11 @@ del_device_store(struct bus_type *bus, const char *buf, size_t count) err = -ENOENT; mutex_lock(&nsim_bus_dev_list_lock); + /* Prevent to use resource before initialization. */ + if (!smp_load_acquire(&nsim_bus_enable)) { + mutex_unlock(&nsim_bus_dev_list_lock); + return -EBUSY; + } list_for_each_entry_safe(nsim_bus_dev, tmp, &nsim_bus_dev_list, list) { if (nsim_bus_dev->dev.id != id) continue; @@ -284,6 +329,9 @@ nsim_bus_dev_new(unsigned int id, unsigned int port_count) nsim_bus_dev->dev.type = &nsim_bus_dev_type; nsim_bus_dev->port_count = port_count; nsim_bus_dev->initial_net = current->nsproxy->net_ns; + mutex_init(&nsim_bus_dev->nsim_bus_reload_lock); + /* Disallow using nsim_bus_dev */ + smp_store_release(&nsim_bus_dev->init, false); err = device_register(&nsim_bus_dev->dev); if (err) @@ -299,6 +347,8 @@ err_nsim_bus_dev_free: static void nsim_bus_dev_del(struct nsim_bus_dev *nsim_bus_dev) { + /* Disallow using nsim_bus_dev */ + smp_store_release(&nsim_bus_dev->init, false); device_unregister(&nsim_bus_dev->dev); ida_free(&nsim_bus_dev_ids, nsim_bus_dev->dev.id); kfree(nsim_bus_dev); @@ -320,6 +370,8 @@ int nsim_bus_init(void) err = driver_register(&nsim_driver); if (err) goto err_bus_unregister; + /* Allow using resources */ + smp_store_release(&nsim_bus_enable, true); return 0; err_bus_unregister: @@ -331,12 +383,16 @@ void nsim_bus_exit(void) { struct nsim_bus_dev *nsim_bus_dev, *tmp; + /* Disallow using resources */ + smp_store_release(&nsim_bus_enable, false); + mutex_lock(&nsim_bus_dev_list_lock); list_for_each_entry_safe(nsim_bus_dev, tmp, &nsim_bus_dev_list, list) { list_del(&nsim_bus_dev->list); nsim_bus_dev_del(nsim_bus_dev); } mutex_unlock(&nsim_bus_dev_list_lock); + driver_unregister(&nsim_driver); bus_unregister(&nsim_bus); } diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index b53fbc06e104..5c5427c840b6 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -73,23 +73,26 @@ static const struct file_operations nsim_dev_take_snapshot_fops = { static int nsim_dev_debugfs_init(struct nsim_dev *nsim_dev) { - char dev_ddir_name[16]; + char dev_ddir_name[sizeof(DRV_NAME) + 10]; sprintf(dev_ddir_name, DRV_NAME "%u", nsim_dev->nsim_bus_dev->dev.id); nsim_dev->ddir = debugfs_create_dir(dev_ddir_name, nsim_dev_ddir); - if (IS_ERR_OR_NULL(nsim_dev->ddir)) - return PTR_ERR_OR_ZERO(nsim_dev->ddir) ?: -EINVAL; + if (IS_ERR(nsim_dev->ddir)) + return PTR_ERR(nsim_dev->ddir); nsim_dev->ports_ddir = debugfs_create_dir("ports", nsim_dev->ddir); - if (IS_ERR_OR_NULL(nsim_dev->ports_ddir)) - return PTR_ERR_OR_ZERO(nsim_dev->ports_ddir) ?: -EINVAL; + if (IS_ERR(nsim_dev->ports_ddir)) + return PTR_ERR(nsim_dev->ports_ddir); debugfs_create_bool("fw_update_status", 0600, nsim_dev->ddir, &nsim_dev->fw_update_status); debugfs_create_u32("max_macs", 0600, nsim_dev->ddir, &nsim_dev->max_macs); debugfs_create_bool("test1", 0600, nsim_dev->ddir, &nsim_dev->test1); - debugfs_create_file("take_snapshot", 0200, nsim_dev->ddir, nsim_dev, - &nsim_dev_take_snapshot_fops); + nsim_dev->take_snapshot = debugfs_create_file("take_snapshot", + 0200, + nsim_dev->ddir, + nsim_dev, + &nsim_dev_take_snapshot_fops); debugfs_create_bool("dont_allow_reload", 0600, nsim_dev->ddir, &nsim_dev->dont_allow_reload); debugfs_create_bool("fail_reload", 0600, nsim_dev->ddir, @@ -112,8 +115,8 @@ static int nsim_dev_port_debugfs_init(struct nsim_dev *nsim_dev, sprintf(port_ddir_name, "%u", nsim_dev_port->port_index); nsim_dev_port->ddir = debugfs_create_dir(port_ddir_name, nsim_dev->ports_ddir); - if (IS_ERR_OR_NULL(nsim_dev_port->ddir)) - return -ENOMEM; + if (IS_ERR(nsim_dev_port->ddir)) + return PTR_ERR(nsim_dev_port->ddir); sprintf(dev_link_name, "../../../" DRV_NAME "%u", nsim_dev->nsim_bus_dev->dev.id); @@ -740,6 +743,11 @@ static int nsim_dev_reload_create(struct nsim_dev *nsim_dev, if (err) goto err_health_exit; + nsim_dev->take_snapshot = debugfs_create_file("take_snapshot", + 0200, + nsim_dev->ddir, + nsim_dev, + &nsim_dev_take_snapshot_fops); return 0; err_health_exit: @@ -853,6 +861,7 @@ static void nsim_dev_reload_destroy(struct nsim_dev *nsim_dev) if (devlink_is_reload_failed(devlink)) return; + debugfs_remove(nsim_dev->take_snapshot); nsim_dev_port_del_all(nsim_dev); nsim_dev_health_exit(nsim_dev); nsim_dev_traps_exit(devlink); @@ -925,8 +934,8 @@ int nsim_dev_port_del(struct nsim_bus_dev *nsim_bus_dev, int nsim_dev_init(void) { nsim_dev_ddir = debugfs_create_dir(DRV_NAME, NULL); - if (IS_ERR_OR_NULL(nsim_dev_ddir)) - return -ENOMEM; + if (IS_ERR(nsim_dev_ddir)) + return PTR_ERR(nsim_dev_ddir); return 0; } diff --git a/drivers/net/netdevsim/health.c b/drivers/net/netdevsim/health.c index 9aa637d162eb..ba8d9ad60feb 100644 --- a/drivers/net/netdevsim/health.c +++ b/drivers/net/netdevsim/health.c @@ -82,7 +82,7 @@ static int nsim_dev_dummy_fmsg_put(struct devlink_fmsg *fmsg, u32 binary_len) if (err) return err; - binary = kmalloc(binary_len, GFP_KERNEL); + binary = kmalloc(binary_len, GFP_KERNEL | __GFP_NOWARN); if (!binary) return -ENOMEM; get_random_bytes(binary, binary_len); @@ -285,8 +285,8 @@ int nsim_dev_health_init(struct nsim_dev *nsim_dev, struct devlink *devlink) } health->ddir = debugfs_create_dir("health", nsim_dev->ddir); - if (IS_ERR_OR_NULL(health->ddir)) { - err = PTR_ERR_OR_ZERO(health->ddir) ?: -EINVAL; + if (IS_ERR(health->ddir)) { + err = PTR_ERR(health->ddir); goto err_dummy_reporter_destroy; } diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h index 94df795ef4d3..2eb7b0dc1594 100644 --- a/drivers/net/netdevsim/netdevsim.h +++ b/drivers/net/netdevsim/netdevsim.h @@ -160,6 +160,7 @@ struct nsim_dev { struct nsim_trap_data *trap_data; struct dentry *ddir; struct dentry *ports_ddir; + struct dentry *take_snapshot; struct bpf_offload_dev *bpf_dev; bool bpf_bind_accept; u32 bpf_bind_verifier_delay; @@ -240,6 +241,9 @@ struct nsim_bus_dev { */ unsigned int num_vfs; struct nsim_vf_config *vfconfigs; + /* Lock for devlink->reload_enabled in netdevsim module */ + struct mutex nsim_bus_reload_lock; + bool init; }; int nsim_bus_init(void); diff --git a/drivers/net/netdevsim/sdev.c b/drivers/net/netdevsim/sdev.c deleted file mode 100644 index 6712da3340d6..000000000000 --- a/drivers/net/netdevsim/sdev.c +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019 Mellanox Technologies. All rights reserved */ - -#include <linux/debugfs.h> -#include <linux/err.h> -#include <linux/kernel.h> -#include <linux/slab.h> - -#include "netdevsim.h" - -static struct dentry *nsim_sdev_ddir; - -static u32 nsim_sdev_id; - -struct netdevsim_shared_dev *nsim_sdev_get(struct netdevsim *joinns) -{ - struct netdevsim_shared_dev *sdev; - char sdev_ddir_name[10]; - int err; - - if (joinns) { - if (WARN_ON(!joinns->sdev)) - return ERR_PTR(-EINVAL); - sdev = joinns->sdev; - sdev->refcnt++; - return sdev; - } - - sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); - if (!sdev) - return ERR_PTR(-ENOMEM); - sdev->refcnt = 1; - sdev->switch_id = nsim_sdev_id++; - - sprintf(sdev_ddir_name, "%u", sdev->switch_id); - sdev->ddir = debugfs_create_dir(sdev_ddir_name, nsim_sdev_ddir); - if (IS_ERR_OR_NULL(sdev->ddir)) { - err = PTR_ERR_OR_ZERO(sdev->ddir) ?: -EINVAL; - goto err_sdev_free; - } - - return sdev; - -err_sdev_free: - nsim_sdev_id--; - kfree(sdev); - return ERR_PTR(err); -} - -void nsim_sdev_put(struct netdevsim_shared_dev *sdev) -{ - if (--sdev->refcnt) - return; - debugfs_remove_recursive(sdev->ddir); - kfree(sdev); -} - -int nsim_sdev_init(void) -{ - nsim_sdev_ddir = debugfs_create_dir(DRV_NAME "_sdev", NULL); - if (IS_ERR_OR_NULL(nsim_sdev_ddir)) - return -ENOMEM; - return 0; -} - -void nsim_sdev_exit(void) -{ - debugfs_remove_recursive(nsim_sdev_ddir); -} diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c index aee62610bade..481cf48c9b9e 100644 --- a/drivers/net/phy/at803x.c +++ b/drivers/net/phy/at803x.c @@ -489,6 +489,14 @@ static int at803x_probe(struct phy_device *phydev) return at803x_parse_dt(phydev); } +static void at803x_remove(struct phy_device *phydev) +{ + struct at803x_priv *priv = phydev->priv; + + if (priv->vddio) + regulator_disable(priv->vddio); +} + static int at803x_clk_out_config(struct phy_device *phydev) { struct at803x_priv *priv = phydev->priv; @@ -711,6 +719,7 @@ static struct phy_driver at803x_driver[] = { .name = "Qualcomm Atheros AR8035", .phy_id_mask = AT803X_PHY_ID_MASK, .probe = at803x_probe, + .remove = at803x_remove, .config_init = at803x_config_init, .set_wol = at803x_set_wol, .get_wol = at803x_get_wol, @@ -726,6 +735,7 @@ static struct phy_driver at803x_driver[] = { .name = "Qualcomm Atheros AR8030", .phy_id_mask = AT803X_PHY_ID_MASK, .probe = at803x_probe, + .remove = at803x_remove, .config_init = at803x_config_init, .link_change_notify = at803x_link_change_notify, .set_wol = at803x_set_wol, @@ -741,6 +751,7 @@ static struct phy_driver at803x_driver[] = { .name = "Qualcomm Atheros AR8031/AR8033", .phy_id_mask = AT803X_PHY_ID_MASK, .probe = at803x_probe, + .remove = at803x_remove, .config_init = at803x_config_init, .set_wol = at803x_set_wol, .get_wol = at803x_get_wol, diff --git a/drivers/net/phy/mdio-mux-meson-g12a.c b/drivers/net/phy/mdio-mux-meson-g12a.c index 7a9ad54582e1..bf86c9c7a288 100644 --- a/drivers/net/phy/mdio-mux-meson-g12a.c +++ b/drivers/net/phy/mdio-mux-meson-g12a.c @@ -123,7 +123,7 @@ static int g12a_ephy_pll_is_enabled(struct clk_hw *hw) return (val & PLL_CTL0_LOCK_DIG) ? 1 : 0; } -static void g12a_ephy_pll_init(struct clk_hw *hw) +static int g12a_ephy_pll_init(struct clk_hw *hw) { struct g12a_ephy_pll *pll = g12a_ephy_pll_to_dev(hw); @@ -136,6 +136,8 @@ static void g12a_ephy_pll_init(struct clk_hw *hw) writel(0x20200000, pll->base + ETH_PLL_CTL5); writel(0x0000c002, pll->base + ETH_PLL_CTL6); writel(0x00000023, pll->base + ETH_PLL_CTL7); + + return 0; } static const struct clk_ops g12a_ephy_pll_ops = { diff --git a/drivers/net/phy/mii_timestamper.c b/drivers/net/phy/mii_timestamper.c index 2f12c5d901df..b71b7456462d 100644 --- a/drivers/net/phy/mii_timestamper.c +++ b/drivers/net/phy/mii_timestamper.c @@ -111,6 +111,13 @@ void unregister_mii_timestamper(struct mii_timestamper *mii_ts) struct mii_timestamping_desc *desc; struct list_head *this; + /* mii_timestamper statically registered by the PHY driver won't use the + * register_mii_timestamper() and thus don't have ->device set. Don't + * try to unregister these. + */ + if (!mii_ts->device) + return; + mutex_lock(&tstamping_devices_lock); list_for_each(this, &mii_timestamping_devices) { desc = list_entry(this, struct mii_timestamping_desc, list); diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index e8cd8c05b156..78ddbaf6401b 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -698,6 +698,9 @@ enum rtl8152_flags { #define VENDOR_ID_NVIDIA 0x0955 #define VENDOR_ID_TPLINK 0x2357 +#define DEVICE_ID_THINKPAD_THUNDERBOLT3_DOCK_GEN2 0x3082 +#define DEVICE_ID_THINKPAD_USB_C_DOCK_GEN2 0xa387 + #define MCU_TYPE_PLA 0x0100 #define MCU_TYPE_USB 0x0000 @@ -6759,9 +6762,13 @@ static int rtl8152_probe(struct usb_interface *intf, netdev->hw_features &= ~NETIF_F_RXCSUM; } - if (le16_to_cpu(udev->descriptor.idVendor) == VENDOR_ID_LENOVO && - le16_to_cpu(udev->descriptor.idProduct) == 0x3082) - set_bit(LENOVO_MACPASSTHRU, &tp->flags); + if (le16_to_cpu(udev->descriptor.idVendor) == VENDOR_ID_LENOVO) { + switch (le16_to_cpu(udev->descriptor.idProduct)) { + case DEVICE_ID_THINKPAD_THUNDERBOLT3_DOCK_GEN2: + case DEVICE_ID_THINKPAD_USB_C_DOCK_GEN2: + set_bit(LENOVO_MACPASSTHRU, &tp->flags); + } + } if (le16_to_cpu(udev->descriptor.bcdDevice) == 0x3011 && udev->serial && (!strcmp(udev->serial, "000001000000") || diff --git a/drivers/net/wireless/cisco/airo.c b/drivers/net/wireless/cisco/airo.c index c4c8f1b62e1e..8363f91df7ea 100644 --- a/drivers/net/wireless/cisco/airo.c +++ b/drivers/net/wireless/cisco/airo.c @@ -4420,73 +4420,65 @@ static int proc_BSSList_open( struct inode *inode, struct file *file ); static int proc_config_open( struct inode *inode, struct file *file ); static int proc_wepkey_open( struct inode *inode, struct file *file ); -static const struct file_operations proc_statsdelta_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .open = proc_statsdelta_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_statsdelta_ops = { + .proc_read = proc_read, + .proc_open = proc_statsdelta_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; -static const struct file_operations proc_stats_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .open = proc_stats_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_stats_ops = { + .proc_read = proc_read, + .proc_open = proc_stats_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; -static const struct file_operations proc_status_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .open = proc_status_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_status_ops = { + .proc_read = proc_read, + .proc_open = proc_status_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; -static const struct file_operations proc_SSID_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .write = proc_write, - .open = proc_SSID_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_SSID_ops = { + .proc_read = proc_read, + .proc_write = proc_write, + .proc_open = proc_SSID_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; -static const struct file_operations proc_BSSList_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .write = proc_write, - .open = proc_BSSList_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_BSSList_ops = { + .proc_read = proc_read, + .proc_write = proc_write, + .proc_open = proc_BSSList_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; -static const struct file_operations proc_APList_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .write = proc_write, - .open = proc_APList_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_APList_ops = { + .proc_read = proc_read, + .proc_write = proc_write, + .proc_open = proc_APList_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; -static const struct file_operations proc_config_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .write = proc_write, - .open = proc_config_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_config_ops = { + .proc_read = proc_read, + .proc_write = proc_write, + .proc_open = proc_config_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; -static const struct file_operations proc_wepkey_ops = { - .owner = THIS_MODULE, - .read = proc_read, - .write = proc_write, - .open = proc_wepkey_open, - .release = proc_close, - .llseek = default_llseek, +static const struct proc_ops proc_wepkey_ops = { + .proc_read = proc_read, + .proc_write = proc_write, + .proc_open = proc_wepkey_open, + .proc_release = proc_close, + .proc_lseek = default_llseek, }; static struct proc_dir_entry *airo_entry; diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_module.c b/drivers/net/wireless/intel/ipw2x00/libipw_module.c index 436b819aeb36..43bab92a4148 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw_module.c +++ b/drivers/net/wireless/intel/ipw2x00/libipw_module.c @@ -240,13 +240,12 @@ static ssize_t debug_level_proc_write(struct file *file, return strnlen(buf, len); } -static const struct file_operations debug_level_proc_fops = { - .owner = THIS_MODULE, - .open = debug_level_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = debug_level_proc_write, +static const struct proc_ops debug_level_proc_ops = { + .proc_open = debug_level_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = debug_level_proc_write, }; #endif /* CONFIG_LIBIPW_DEBUG */ @@ -263,7 +262,7 @@ static int __init libipw_init(void) return -EIO; } e = proc_create("debug_level", 0644, libipw_proc, - &debug_level_proc_fops); + &debug_level_proc_ops); if (!e) { remove_proc_entry(DRV_PROCNAME, init_net.proc_net); libipw_proc = NULL; diff --git a/drivers/net/wireless/intersil/hostap/hostap_hw.c b/drivers/net/wireless/intersil/hostap/hostap_hw.c index e323e9a5999f..58212c532c90 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_hw.c +++ b/drivers/net/wireless/intersil/hostap/hostap_hw.c @@ -126,7 +126,7 @@ static void prism2_check_sta_fw_version(local_info_t *local); #ifdef PRISM2_DOWNLOAD_SUPPORT /* hostap_download.c */ -static const struct file_operations prism2_download_aux_dump_proc_fops; +static const struct proc_ops prism2_download_aux_dump_proc_ops; static u8 * prism2_read_pda(struct net_device *dev); static int prism2_download(local_info_t *local, struct prism2_download_param *param); @@ -3094,7 +3094,7 @@ prism2_init_local_data(struct prism2_helper_functions *funcs, int card_idx, local->func->reset_port = prism2_reset_port; local->func->schedule_reset = prism2_schedule_reset; #ifdef PRISM2_DOWNLOAD_SUPPORT - local->func->read_aux_fops = &prism2_download_aux_dump_proc_fops; + local->func->read_aux_proc_ops = &prism2_download_aux_dump_proc_ops; local->func->download = prism2_download; #endif /* PRISM2_DOWNLOAD_SUPPORT */ local->func->tx = prism2_tx_80211; diff --git a/drivers/net/wireless/intersil/hostap/hostap_proc.c b/drivers/net/wireless/intersil/hostap/hostap_proc.c index 6151d8db5924..a2ee4693eaed 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_proc.c +++ b/drivers/net/wireless/intersil/hostap/hostap_proc.c @@ -211,9 +211,9 @@ static ssize_t prism2_pda_proc_read(struct file *file, char __user *buf, return count; } -static const struct file_operations prism2_pda_proc_fops = { - .read = prism2_pda_proc_read, - .llseek = generic_file_llseek, +static const struct proc_ops prism2_pda_proc_ops = { + .proc_read = prism2_pda_proc_read, + .proc_lseek = generic_file_llseek, }; @@ -223,8 +223,8 @@ static ssize_t prism2_aux_dump_proc_no_read(struct file *file, char __user *buf, return 0; } -static const struct file_operations prism2_aux_dump_proc_fops = { - .read = prism2_aux_dump_proc_no_read, +static const struct proc_ops prism2_aux_dump_proc_ops = { + .proc_read = prism2_aux_dump_proc_no_read, }; @@ -379,9 +379,9 @@ void hostap_init_proc(local_info_t *local) proc_create_seq_data("wds", 0, local->proc, &prism2_wds_proc_seqops, local); proc_create_data("pda", 0, local->proc, - &prism2_pda_proc_fops, local); + &prism2_pda_proc_ops, local); proc_create_data("aux_dump", 0, local->proc, - local->func->read_aux_fops ?: &prism2_aux_dump_proc_fops, + local->func->read_aux_proc_ops ?: &prism2_aux_dump_proc_ops, local); proc_create_seq_data("bss_list", 0, local->proc, &prism2_bss_list_proc_seqops, local); diff --git a/drivers/net/wireless/intersil/hostap/hostap_wlan.h b/drivers/net/wireless/intersil/hostap/hostap_wlan.h index a8c4c1a8b29d..487883fbb58c 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_wlan.h +++ b/drivers/net/wireless/intersil/hostap/hostap_wlan.h @@ -599,7 +599,7 @@ struct prism2_helper_functions { struct prism2_download_param *param); int (*tx)(struct sk_buff *skb, struct net_device *dev); int (*set_tim)(struct net_device *dev, int aid, int set); - const struct file_operations *read_aux_fops; + const struct proc_ops *read_aux_proc_ops; int need_tx_headroom; /* number of bytes of headroom needed before * IEEE 802.11 header */ diff --git a/drivers/net/wireless/ray_cs.c b/drivers/net/wireless/ray_cs.c index cf372684b681..c1d542bfa530 100644 --- a/drivers/net/wireless/ray_cs.c +++ b/drivers/net/wireless/ray_cs.c @@ -2717,10 +2717,9 @@ static ssize_t ray_cs_essid_proc_write(struct file *file, return count; } -static const struct file_operations ray_cs_essid_proc_fops = { - .owner = THIS_MODULE, - .write = ray_cs_essid_proc_write, - .llseek = noop_llseek, +static const struct proc_ops ray_cs_essid_proc_ops = { + .proc_write = ray_cs_essid_proc_write, + .proc_lseek = noop_llseek, }; static ssize_t int_proc_write(struct file *file, const char __user *buffer, @@ -2751,10 +2750,9 @@ static ssize_t int_proc_write(struct file *file, const char __user *buffer, return count; } -static const struct file_operations int_proc_fops = { - .owner = THIS_MODULE, - .write = int_proc_write, - .llseek = noop_llseek, +static const struct proc_ops int_proc_ops = { + .proc_write = int_proc_write, + .proc_lseek = noop_llseek, }; #endif @@ -2790,10 +2788,10 @@ static int __init init_ray_cs(void) proc_mkdir("driver/ray_cs", NULL); proc_create_single("driver/ray_cs/ray_cs", 0, NULL, ray_cs_proc_show); - proc_create("driver/ray_cs/essid", 0200, NULL, &ray_cs_essid_proc_fops); - proc_create_data("driver/ray_cs/net_type", 0200, NULL, &int_proc_fops, + proc_create("driver/ray_cs/essid", 0200, NULL, &ray_cs_essid_proc_ops); + proc_create_data("driver/ray_cs/net_type", 0200, NULL, &int_proc_ops, &net_type); - proc_create_data("driver/ray_cs/translate", 0200, NULL, &int_proc_fops, + proc_create_data("driver/ray_cs/translate", 0200, NULL, &int_proc_ops, &translate); #endif if (translate != 0) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 365a2ddbeaa7..da392b50f73e 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -167,7 +167,6 @@ struct nvme_queue { /* only used for poll queues: */ spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; volatile struct nvme_completion *cqes; - struct blk_mq_tags **tags; dma_addr_t sq_dma_addr; dma_addr_t cq_dma_addr; u32 __iomem *q_db; @@ -376,29 +375,17 @@ static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, WARN_ON(hctx_idx != 0); WARN_ON(dev->admin_tagset.tags[0] != hctx->tags); - WARN_ON(nvmeq->tags); hctx->driver_data = nvmeq; - nvmeq->tags = &dev->admin_tagset.tags[0]; return 0; } -static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) -{ - struct nvme_queue *nvmeq = hctx->driver_data; - - nvmeq->tags = NULL; -} - static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { struct nvme_dev *dev = data; struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1]; - if (!nvmeq->tags) - nvmeq->tags = &dev->tagset.tags[hctx_idx]; - WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags); hctx->driver_data = nvmeq; return 0; @@ -948,6 +935,13 @@ static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq) writel(head, nvmeq->q_db + nvmeq->dev->db_stride); } +static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq) +{ + if (!nvmeq->qid) + return nvmeq->dev->admin_tagset.tags[0]; + return nvmeq->dev->tagset.tags[nvmeq->qid - 1]; +} + static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) { volatile struct nvme_completion *cqe = &nvmeq->cqes[idx]; @@ -972,7 +966,7 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) return; } - req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id); + req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), cqe->command_id); trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); nvme_end_request(req, cqe->status, cqe->result); } @@ -1572,7 +1566,6 @@ static const struct blk_mq_ops nvme_mq_admin_ops = { .queue_rq = nvme_queue_rq, .complete = nvme_pci_complete_rq, .init_hctx = nvme_admin_init_hctx, - .exit_hctx = nvme_admin_exit_hctx, .init_request = nvme_init_request, .timeout = nvme_timeout, }; diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 28438b833c1b..576de773b4db 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -129,27 +129,8 @@ static u32 nvmet_async_event_result(struct nvmet_async_event *aen) return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16); } -static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) -{ - struct nvmet_req *req; - - while (1) { - mutex_lock(&ctrl->lock); - if (!ctrl->nr_async_event_cmds) { - mutex_unlock(&ctrl->lock); - return; - } - - req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; - mutex_unlock(&ctrl->lock); - nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR); - } -} - -static void nvmet_async_event_work(struct work_struct *work) +static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status) { - struct nvmet_ctrl *ctrl = - container_of(work, struct nvmet_ctrl, async_event_work); struct nvmet_async_event *aen; struct nvmet_req *req; @@ -159,18 +140,41 @@ static void nvmet_async_event_work(struct work_struct *work) struct nvmet_async_event, entry); if (!aen || !ctrl->nr_async_event_cmds) { mutex_unlock(&ctrl->lock); - return; + break; } req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; - nvmet_set_result(req, nvmet_async_event_result(aen)); + if (status == 0) + nvmet_set_result(req, nvmet_async_event_result(aen)); list_del(&aen->entry); kfree(aen); mutex_unlock(&ctrl->lock); - nvmet_req_complete(req, 0); + nvmet_req_complete(req, status); + } +} + +static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) +{ + struct nvmet_req *req; + + mutex_lock(&ctrl->lock); + while (ctrl->nr_async_event_cmds) { + req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; + mutex_unlock(&ctrl->lock); + nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR); + mutex_lock(&ctrl->lock); } + mutex_unlock(&ctrl->lock); +} + +static void nvmet_async_event_work(struct work_struct *work) +{ + struct nvmet_ctrl *ctrl = + container_of(work, struct nvmet_ctrl, async_event_work); + + nvmet_async_events_process(ctrl, 0); } void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, @@ -555,7 +559,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns) } else { struct nvmet_ns *old; - list_for_each_entry_rcu(old, &subsys->namespaces, dev_link) { + list_for_each_entry_rcu(old, &subsys->namespaces, dev_link, + lockdep_is_held(&subsys->lock)) { BUG_ON(ns->nsid == old->nsid); if (ns->nsid < old->nsid) break; @@ -752,19 +757,24 @@ static void nvmet_confirm_sq(struct percpu_ref *ref) void nvmet_sq_destroy(struct nvmet_sq *sq) { + u16 status = NVME_SC_INTERNAL | NVME_SC_DNR; + struct nvmet_ctrl *ctrl = sq->ctrl; + /* * If this is the admin queue, complete all AERs so that our * queue doesn't have outstanding requests on it. */ - if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq) - nvmet_async_events_free(sq->ctrl); + if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq) { + nvmet_async_events_process(ctrl, status); + nvmet_async_events_free(ctrl); + } percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq); wait_for_completion(&sq->confirm_done); wait_for_completion(&sq->free_done); percpu_ref_exit(&sq->ref); - if (sq->ctrl) { - nvmet_ctrl_put(sq->ctrl); + if (ctrl) { + nvmet_ctrl_put(ctrl); sq->ctrl = NULL; /* allows reusing the queue later */ } } @@ -938,6 +948,17 @@ bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len) } EXPORT_SYMBOL_GPL(nvmet_check_data_len); +bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len) +{ + if (unlikely(data_len > req->transfer_len)) { + req->error_loc = offsetof(struct nvme_common_command, dptr); + nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR); + return false; + } + + return true; +} + int nvmet_req_alloc_sgl(struct nvmet_req *req) { struct pci_dev *p2p_dev = NULL; @@ -1172,7 +1193,8 @@ static void nvmet_setup_p2p_ns_map(struct nvmet_ctrl *ctrl, ctrl->p2p_client = get_device(req->p2p_client); - list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) + list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link, + lockdep_is_held(&ctrl->subsys->lock)) nvmet_p2pmem_ns_add_p2p(ctrl, ns); } diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index f7297473d9eb..feef15c38ec9 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -109,6 +109,7 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req) u16 qid = le16_to_cpu(c->qid); u16 sqsize = le16_to_cpu(c->sqsize); struct nvmet_ctrl *old; + u16 ret; old = cmpxchg(&req->sq->ctrl, NULL, ctrl); if (old) { @@ -119,7 +120,8 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req) if (!sqsize) { pr_warn("queue size zero!\n"); req->error_loc = offsetof(struct nvmf_connect_command, sqsize); - return NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; + ret = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; + goto err; } /* note: convert queue size from 0's-based value to 1's-based value */ @@ -132,16 +134,19 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req) } if (ctrl->ops->install_queue) { - u16 ret = ctrl->ops->install_queue(req->sq); - + ret = ctrl->ops->install_queue(req->sq); if (ret) { pr_err("failed to install queue %d cntlid %d ret %x\n", - qid, ret, ctrl->cntlid); - return ret; + qid, ctrl->cntlid, ret); + goto err; } } return 0; + +err: + req->sq->ctrl = NULL; + return ret; } static void nvmet_execute_admin_connect(struct nvmet_req *req) diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index b6fca0e421ef..ea0e596be15d 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -280,7 +280,7 @@ static void nvmet_bdev_execute_discard(struct nvmet_req *req) static void nvmet_bdev_execute_dsm(struct nvmet_req *req) { - if (!nvmet_check_data_len(req, nvmet_dsm_len(req))) + if (!nvmet_check_data_len_lte(req, nvmet_dsm_len(req))) return; switch (le32_to_cpu(req->cmd->dsm.attributes)) { diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index caebfce06605..cd5670b83118 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -336,7 +336,7 @@ static void nvmet_file_dsm_work(struct work_struct *w) static void nvmet_file_execute_dsm(struct nvmet_req *req) { - if (!nvmet_check_data_len(req, nvmet_dsm_len(req))) + if (!nvmet_check_data_len_lte(req, nvmet_dsm_len(req))) return; INIT_WORK(&req->f.work, nvmet_file_dsm_work); schedule_work(&req->f.work); diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 46df45e837c9..eda28b22a2c8 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -374,6 +374,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq, struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops); void nvmet_req_uninit(struct nvmet_req *req); bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len); +bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len); void nvmet_req_complete(struct nvmet_req *req, u16 status); int nvmet_req_alloc_sgl(struct nvmet_req *req); void nvmet_req_free_sgl(struct nvmet_req *req); diff --git a/drivers/of/device.c b/drivers/of/device.c index e9127db7b067..27203bfd0b22 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -161,7 +161,7 @@ int of_dma_configure(struct device *dev, struct device_node *np, bool force_dma) coherent ? " " : " not "); iommu = of_iommu_configure(dev, np); - if (IS_ERR(iommu) && PTR_ERR(iommu) == -EPROBE_DEFER) + if (PTR_ERR(iommu) == -EPROBE_DEFER) return -EPROBE_DEFER; dev_dbg(dev, "device is%sbehind an iommu\n", diff --git a/drivers/of/of_mdio.c b/drivers/of/of_mdio.c index f5c2a5487761..8270bbf505fb 100644 --- a/drivers/of/of_mdio.c +++ b/drivers/of/of_mdio.c @@ -81,13 +81,15 @@ static int of_mdiobus_register_phy(struct mii_bus *mdio, else phy = get_phy_device(mdio, addr, is_c45); if (IS_ERR(phy)) { - unregister_mii_timestamper(mii_ts); + if (mii_ts) + unregister_mii_timestamper(mii_ts); return PTR_ERR(phy); } rc = of_irq_get(child, 0); if (rc == -EPROBE_DEFER) { - unregister_mii_timestamper(mii_ts); + if (mii_ts) + unregister_mii_timestamper(mii_ts); phy_device_free(phy); return rc; } @@ -116,12 +118,19 @@ static int of_mdiobus_register_phy(struct mii_bus *mdio, * register it */ rc = phy_device_register(phy); if (rc) { - unregister_mii_timestamper(mii_ts); + if (mii_ts) + unregister_mii_timestamper(mii_ts); phy_device_free(phy); of_node_put(child); return rc; } - phy->mii_ts = mii_ts; + + /* phy->mii_ts may already be defined by the PHY driver. A + * mii_timestamper probed via the device tree will still have + * precedence. + */ + if (mii_ts) + phy->mii_ts = mii_ts; dev_dbg(&mdio->dev, "registered phy %pOFn at address %i\n", child, addr); diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index eda2633a393d..9210a95cb4e6 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -32,7 +32,7 @@ #define OP_BUFFER_FLAGS 0 -static struct ring_buffer *op_ring_buffer; +static struct trace_buffer *op_ring_buffer; DEFINE_PER_CPU(struct oprofile_cpu_buffer, op_cpu_buffer); static void wq_sync_buffer(struct work_struct *work); diff --git a/drivers/parisc/led.c b/drivers/parisc/led.c index 73e37bb877a4..36c6613f7a36 100644 --- a/drivers/parisc/led.c +++ b/drivers/parisc/led.c @@ -230,13 +230,12 @@ parse_error: return -EINVAL; } -static const struct file_operations led_proc_fops = { - .owner = THIS_MODULE, - .open = led_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = led_proc_write, +static const struct proc_ops led_proc_ops = { + .proc_open = led_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = led_proc_write, }; static int __init led_create_procfs(void) @@ -252,14 +251,14 @@ static int __init led_create_procfs(void) if (!lcd_no_led_support) { ent = proc_create_data("led", S_IRUGO|S_IWUSR, proc_pdc_root, - &led_proc_fops, (void *)LED_NOLCD); /* LED */ + &led_proc_ops, (void *)LED_NOLCD); /* LED */ if (!ent) return -1; } if (led_type == LED_HASLCD) { ent = proc_create_data("lcd", S_IRUGO|S_IWUSR, proc_pdc_root, - &led_proc_fops, (void *)LED_HASLCD); /* LCD */ + &led_proc_ops, (void *)LED_HASLCD); /* LCD */ if (!ent) return -1; } diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index 982b46f0a54d..3ef0bb281e7c 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -69,6 +69,7 @@ int pci_enable_ats(struct pci_dev *dev, int ps) dev->ats_enabled = 1; return 0; } +EXPORT_SYMBOL_GPL(pci_enable_ats); /** * pci_disable_ats - disable the ATS capability @@ -87,6 +88,7 @@ void pci_disable_ats(struct pci_dev *dev) dev->ats_enabled = 0; } +EXPORT_SYMBOL_GPL(pci_disable_ats); void pci_restore_ats_state(struct pci_dev *dev) { @@ -424,11 +426,12 @@ void pci_restore_pasid_state(struct pci_dev *pdev) int pci_pasid_features(struct pci_dev *pdev) { u16 supported; - int pasid = pdev->pasid_cap; + int pasid; if (pdev->is_virtfn) pdev = pci_physfn(pdev); + pasid = pdev->pasid_cap; if (!pasid) return -EINVAL; @@ -451,11 +454,12 @@ int pci_pasid_features(struct pci_dev *pdev) int pci_max_pasids(struct pci_dev *pdev) { u16 supported; - int pasid = pdev->pasid_cap; + int pasid; if (pdev->is_virtfn) pdev = pci_physfn(pdev); + pasid = pdev->pasid_cap; if (!pasid) return -EINVAL; diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index c77069c8ee5d..20bf00f587bd 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -239,7 +239,6 @@ config PCIE_TANGO_SMP8759 config VMD depends on PCI_MSI && X86_64 && SRCU - select X86_DEV_DMA_OPS tristate "Intel Volume Management Device Driver" ---help--- Adds support for the Intel Volume Management Device (VMD). VMD is a @@ -253,6 +252,15 @@ config VMD To compile this driver as a module, choose M here: the module will be called vmd. +config PCIE_BRCMSTB + tristate "Broadcom Brcmstb PCIe host controller" + depends on ARCH_BCM2835 || COMPILE_TEST + depends on OF + depends on PCI_MSI_IRQ_DOMAIN + help + Say Y here to enable PCIe host controller support for + Broadcom STB based SoCs, like the Raspberry Pi 4. + config PCI_HYPERV_INTERFACE tristate "Hyper-V PCI Interface" depends on X86 && HYPERV && PCI_MSI && PCI_MSI_IRQ_DOMAIN && X86_64 diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 3d4f597f15ce..01b2502a5323 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -28,6 +28,7 @@ obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o obj-$(CONFIG_PCIE_MOBIVEIL) += pcie-mobiveil.o obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o obj-$(CONFIG_VMD) += vmd.o +obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW obj-y += dwc/ diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index 625a031b2193..0830dfcfa43a 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -209,6 +209,17 @@ config PCIE_ARTPEC6_EP Enables support for the PCIe controller in the ARTPEC-6 SoC to work in endpoint mode. This uses the DesignWare core. +config PCIE_INTEL_GW + bool "Intel Gateway PCIe host controller support" + depends on OF && (X86 || COMPILE_TEST) + depends on PCI_MSI_IRQ_DOMAIN + select PCIE_DW_HOST + help + Say 'Y' here to enable PCIe Host controller support on Intel + Gateway SoCs. + The PCIe controller uses the DesignWare core plus Intel-specific + hardware wrappers. + config PCIE_KIRIN depends on OF && (ARM64 || COMPILE_TEST) bool "HiSilicon Kirin series SoCs PCIe controllers" diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile index 69faff371f11..8a637cfcf6e9 100644 --- a/drivers/pci/controller/dwc/Makefile +++ b/drivers/pci/controller/dwc/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_PCI_LAYERSCAPE_EP) += pci-layerscape-ep.o obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o obj-$(CONFIG_PCIE_ARTPEC6) += pcie-artpec6.o +obj-$(CONFIG_PCIE_INTEL_GW) += pcie-intel-gw.o obj-$(CONFIG_PCIE_KIRIN) += pcie-kirin.o obj-$(CONFIG_PCIE_HISI_STB) += pcie-histb.o obj-$(CONFIG_PCI_MESON) += pci-meson.o diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c index 14a6ba4067fb..c5043d951e80 100644 --- a/drivers/pci/controller/dwc/pci-exynos.c +++ b/drivers/pci/controller/dwc/pci-exynos.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * PCIe host controller driver for Samsung EXYNOS SoCs + * PCIe host controller driver for Samsung Exynos SoCs * * Copyright (C) 2013 Samsung Electronics Co., Ltd. * http://www.samsung.com diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index af677254a072..c8c702c494a2 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -422,7 +422,7 @@ static void ks_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie) lower_32_bits(start) | OB_ENABLEN); ks_pcie_app_writel(ks_pcie, OB_OFFSET_HI(i), upper_32_bits(start)); - start += OB_WIN_SIZE; + start += OB_WIN_SIZE * SZ_1M; } val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); @@ -510,7 +510,7 @@ static void ks_pcie_stop_link(struct dw_pcie *pci) /* Disable Link training */ val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); val &= ~LTSSM_EN_VAL; - ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val); + ks_pcie_app_writel(ks_pcie, CMD_STATUS, val); } static int ks_pcie_start_link(struct dw_pcie *pci) @@ -1354,7 +1354,7 @@ static int __init ks_pcie_probe(struct platform_device *pdev) ret = of_property_read_u32(np, "num-viewport", &num_viewport); if (ret < 0) { dev_err(dev, "unable to read *num-viewport* property\n"); - return ret; + goto err_get_sync; } /* diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c index 9e2482bd7b6d..28d5a1095200 100644 --- a/drivers/pci/controller/dwc/pcie-artpec6.c +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -51,9 +51,6 @@ static const struct of_device_id artpec6_pcie_of_match[]; #define ACK_N_FTS_MASK GENMASK(15, 8) #define ACK_N_FTS(x) (((x) << 8) & ACK_N_FTS_MASK) -#define FAST_TRAINING_SEQ_MASK GENMASK(7, 0) -#define FAST_TRAINING_SEQ(x) (((x) << 0) & FAST_TRAINING_SEQ_MASK) - /* ARTPEC-6 specific registers */ #define PCIECFG 0x18 #define PCIECFG_DBG_OEN BIT(24) @@ -313,10 +310,7 @@ static void artpec6_pcie_set_nfts(struct artpec6_pcie *artpec6_pcie) * Set the Number of Fast Training Sequences that the core * advertises as its N_FTS during Gen2 or Gen3 link training. */ - val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); - val &= ~FAST_TRAINING_SEQ_MASK; - val |= FAST_TRAINING_SEQ(180); - dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + dw_pcie_link_set_n_fts(pci, 180); } static void artpec6_pcie_assert_core_reset(struct artpec6_pcie *artpec6_pcie) diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 820488dfeaed..681548c88282 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -12,6 +12,7 @@ #include <linux/of.h> #include <linux/types.h> +#include "../../pci.h" #include "pcie-designware.h" /* @@ -474,6 +475,61 @@ int dw_pcie_link_up(struct dw_pcie *pci) (!(val & PCIE_PORT_DEBUG1_LINK_IN_TRAINING))); } +void dw_pcie_upconfig_setup(struct dw_pcie *pci) +{ + u32 val; + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL); + val |= PORT_MLTI_UPCFG_SUPPORT; + dw_pcie_writel_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL, val); +} +EXPORT_SYMBOL_GPL(dw_pcie_upconfig_setup); + +void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen) +{ + u32 reg, val; + u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + reg = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCTL2); + reg &= ~PCI_EXP_LNKCTL2_TLS; + + switch (pcie_link_speed[link_gen]) { + case PCIE_SPEED_2_5GT: + reg |= PCI_EXP_LNKCTL2_TLS_2_5GT; + break; + case PCIE_SPEED_5_0GT: + reg |= PCI_EXP_LNKCTL2_TLS_5_0GT; + break; + case PCIE_SPEED_8_0GT: + reg |= PCI_EXP_LNKCTL2_TLS_8_0GT; + break; + case PCIE_SPEED_16_0GT: + reg |= PCI_EXP_LNKCTL2_TLS_16_0GT; + break; + default: + /* Use hardware capability */ + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + val = FIELD_GET(PCI_EXP_LNKCAP_SLS, val); + reg &= ~PCI_EXP_LNKCTL2_HASD; + reg |= FIELD_PREP(PCI_EXP_LNKCTL2_TLS, val); + break; + } + + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCTL2, reg); +} +EXPORT_SYMBOL_GPL(dw_pcie_link_set_max_speed); + +void dw_pcie_link_set_n_fts(struct dw_pcie *pci, u32 n_fts) +{ + u32 val; + + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val &= ~PORT_LOGIC_N_FTS_MASK; + val |= n_fts & PORT_LOGIC_N_FTS_MASK; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); +} +EXPORT_SYMBOL_GPL(dw_pcie_link_set_n_fts); + static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci) { u32 val; diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 5accdd6bc388..a22ea5982817 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -30,7 +30,12 @@ #define LINK_WAIT_IATU 9 /* Synopsys-specific PCIe configuration registers */ +#define PCIE_PORT_AFR 0x70C +#define PORT_AFR_N_FTS_MASK GENMASK(15, 8) +#define PORT_AFR_CC_N_FTS_MASK GENMASK(23, 16) + #define PCIE_PORT_LINK_CONTROL 0x710 +#define PORT_LINK_DLL_LINK_EN BIT(5) #define PORT_LINK_MODE_MASK GENMASK(21, 16) #define PORT_LINK_MODE(n) FIELD_PREP(PORT_LINK_MODE_MASK, n) #define PORT_LINK_MODE_1_LANES PORT_LINK_MODE(0x1) @@ -46,6 +51,7 @@ #define PCIE_PORT_DEBUG1_LINK_IN_TRAINING BIT(29) #define PCIE_LINK_WIDTH_SPEED_CONTROL 0x80C +#define PORT_LOGIC_N_FTS_MASK GENMASK(7, 0) #define PORT_LOGIC_SPEED_CHANGE BIT(17) #define PORT_LOGIC_LINK_WIDTH_MASK GENMASK(12, 8) #define PORT_LOGIC_LINK_WIDTH(n) FIELD_PREP(PORT_LOGIC_LINK_WIDTH_MASK, n) @@ -60,6 +66,9 @@ #define PCIE_MSI_INTR0_MASK 0x82C #define PCIE_MSI_INTR0_STATUS 0x830 +#define PCIE_PORT_MULTI_LANE_CTRL 0x8C0 +#define PORT_MLTI_UPCFG_SUPPORT BIT(7) + #define PCIE_ATU_VIEWPORT 0x900 #define PCIE_ATU_REGION_INBOUND BIT(31) #define PCIE_ATU_REGION_OUTBOUND 0 @@ -273,6 +282,9 @@ void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val); u32 dw_pcie_read_atu(struct dw_pcie *pci, u32 reg, size_t size); void dw_pcie_write_atu(struct dw_pcie *pci, u32 reg, size_t size, u32 val); int dw_pcie_link_up(struct dw_pcie *pci); +void dw_pcie_upconfig_setup(struct dw_pcie *pci); +void dw_pcie_link_set_max_speed(struct dw_pcie *pci, u32 link_gen); +void dw_pcie_link_set_n_fts(struct dw_pcie *pci, u32 n_fts); int dw_pcie_wait_for_link(struct dw_pcie *pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, u64 cpu_addr, u64 pci_addr, diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c new file mode 100644 index 000000000000..fc2a12212dec --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Intel Gateway SoCs + * + * Copyright (c) 2019 Intel Corporation. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/gpio/consumer.h> +#include <linux/iopoll.h> +#include <linux/pci_regs.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +#include "../../pci.h" +#include "pcie-designware.h" + +#define PORT_AFR_N_FTS_GEN12_DFT (SZ_128 - 1) +#define PORT_AFR_N_FTS_GEN3 180 +#define PORT_AFR_N_FTS_GEN4 196 + +/* PCIe Application logic Registers */ +#define PCIE_APP_CCR 0x10 +#define PCIE_APP_CCR_LTSSM_ENABLE BIT(0) + +#define PCIE_APP_MSG_CR 0x30 +#define PCIE_APP_MSG_XMT_PM_TURNOFF BIT(0) + +#define PCIE_APP_PMC 0x44 +#define PCIE_APP_PMC_IN_L2 BIT(20) + +#define PCIE_APP_IRNEN 0xF4 +#define PCIE_APP_IRNCR 0xF8 +#define PCIE_APP_IRN_AER_REPORT BIT(0) +#define PCIE_APP_IRN_PME BIT(2) +#define PCIE_APP_IRN_RX_VDM_MSG BIT(4) +#define PCIE_APP_IRN_PM_TO_ACK BIT(9) +#define PCIE_APP_IRN_LINK_AUTO_BW_STAT BIT(11) +#define PCIE_APP_IRN_BW_MGT BIT(12) +#define PCIE_APP_IRN_MSG_LTR BIT(18) +#define PCIE_APP_IRN_SYS_ERR_RC BIT(29) +#define PCIE_APP_INTX_OFST 12 + +#define PCIE_APP_IRN_INT \ + (PCIE_APP_IRN_AER_REPORT | PCIE_APP_IRN_PME | \ + PCIE_APP_IRN_RX_VDM_MSG | PCIE_APP_IRN_SYS_ERR_RC | \ + PCIE_APP_IRN_PM_TO_ACK | PCIE_APP_IRN_MSG_LTR | \ + PCIE_APP_IRN_BW_MGT | PCIE_APP_IRN_LINK_AUTO_BW_STAT | \ + (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTA) | \ + (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTB) | \ + (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTC) | \ + (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTD)) + +#define BUS_IATU_OFFSET SZ_256M +#define RESET_INTERVAL_MS 100 + +struct intel_pcie_soc { + unsigned int pcie_ver; + unsigned int pcie_atu_offset; + u32 num_viewport; +}; + +struct intel_pcie_port { + struct dw_pcie pci; + void __iomem *app_base; + struct gpio_desc *reset_gpio; + u32 rst_intrvl; + u32 max_speed; + u32 link_gen; + u32 max_width; + u32 n_fts; + struct clk *core_clk; + struct reset_control *core_rst; + struct phy *phy; + u8 pcie_cap_ofst; +}; + +static void pcie_update_bits(void __iomem *base, u32 ofs, u32 mask, u32 val) +{ + u32 old; + + old = readl(base + ofs); + val = (old & ~mask) | (val & mask); + + if (val != old) + writel(val, base + ofs); +} + +static inline u32 pcie_app_rd(struct intel_pcie_port *lpp, u32 ofs) +{ + return readl(lpp->app_base + ofs); +} + +static inline void pcie_app_wr(struct intel_pcie_port *lpp, u32 ofs, u32 val) +{ + writel(val, lpp->app_base + ofs); +} + +static void pcie_app_wr_mask(struct intel_pcie_port *lpp, u32 ofs, + u32 mask, u32 val) +{ + pcie_update_bits(lpp->app_base, ofs, mask, val); +} + +static inline u32 pcie_rc_cfg_rd(struct intel_pcie_port *lpp, u32 ofs) +{ + return dw_pcie_readl_dbi(&lpp->pci, ofs); +} + +static inline void pcie_rc_cfg_wr(struct intel_pcie_port *lpp, u32 ofs, u32 val) +{ + dw_pcie_writel_dbi(&lpp->pci, ofs, val); +} + +static void pcie_rc_cfg_wr_mask(struct intel_pcie_port *lpp, u32 ofs, + u32 mask, u32 val) +{ + pcie_update_bits(lpp->pci.dbi_base, ofs, mask, val); +} + +static void intel_pcie_ltssm_enable(struct intel_pcie_port *lpp) +{ + pcie_app_wr_mask(lpp, PCIE_APP_CCR, PCIE_APP_CCR_LTSSM_ENABLE, + PCIE_APP_CCR_LTSSM_ENABLE); +} + +static void intel_pcie_ltssm_disable(struct intel_pcie_port *lpp) +{ + pcie_app_wr_mask(lpp, PCIE_APP_CCR, PCIE_APP_CCR_LTSSM_ENABLE, 0); +} + +static void intel_pcie_link_setup(struct intel_pcie_port *lpp) +{ + u32 val; + u8 offset = lpp->pcie_cap_ofst; + + val = pcie_rc_cfg_rd(lpp, offset + PCI_EXP_LNKCAP); + lpp->max_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, val); + lpp->max_width = FIELD_GET(PCI_EXP_LNKCAP_MLW, val); + + val = pcie_rc_cfg_rd(lpp, offset + PCI_EXP_LNKCTL); + + val &= ~(PCI_EXP_LNKCTL_LD | PCI_EXP_LNKCTL_ASPMC); + pcie_rc_cfg_wr(lpp, offset + PCI_EXP_LNKCTL, val); +} + +static void intel_pcie_port_logic_setup(struct intel_pcie_port *lpp) +{ + u32 val, mask; + + switch (pcie_link_speed[lpp->max_speed]) { + case PCIE_SPEED_8_0GT: + lpp->n_fts = PORT_AFR_N_FTS_GEN3; + break; + case PCIE_SPEED_16_0GT: + lpp->n_fts = PORT_AFR_N_FTS_GEN4; + break; + default: + lpp->n_fts = PORT_AFR_N_FTS_GEN12_DFT; + break; + } + + mask = PORT_AFR_N_FTS_MASK | PORT_AFR_CC_N_FTS_MASK; + val = FIELD_PREP(PORT_AFR_N_FTS_MASK, lpp->n_fts) | + FIELD_PREP(PORT_AFR_CC_N_FTS_MASK, lpp->n_fts); + pcie_rc_cfg_wr_mask(lpp, PCIE_PORT_AFR, mask, val); + + /* Port Link Control Register */ + pcie_rc_cfg_wr_mask(lpp, PCIE_PORT_LINK_CONTROL, PORT_LINK_DLL_LINK_EN, + PORT_LINK_DLL_LINK_EN); +} + +static void intel_pcie_rc_setup(struct intel_pcie_port *lpp) +{ + intel_pcie_ltssm_disable(lpp); + intel_pcie_link_setup(lpp); + dw_pcie_setup_rc(&lpp->pci.pp); + dw_pcie_upconfig_setup(&lpp->pci); + intel_pcie_port_logic_setup(lpp); + dw_pcie_link_set_max_speed(&lpp->pci, lpp->link_gen); + dw_pcie_link_set_n_fts(&lpp->pci, lpp->n_fts); +} + +static int intel_pcie_ep_rst_init(struct intel_pcie_port *lpp) +{ + struct device *dev = lpp->pci.dev; + int ret; + + lpp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(lpp->reset_gpio)) { + ret = PTR_ERR(lpp->reset_gpio); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to request PCIe GPIO: %d\n", ret); + return ret; + } + + /* Make initial reset last for 100us */ + usleep_range(100, 200); + + return 0; +} + +static void intel_pcie_core_rst_assert(struct intel_pcie_port *lpp) +{ + reset_control_assert(lpp->core_rst); +} + +static void intel_pcie_core_rst_deassert(struct intel_pcie_port *lpp) +{ + /* + * One micro-second delay to make sure the reset pulse + * wide enough so that core reset is clean. + */ + udelay(1); + reset_control_deassert(lpp->core_rst); + + /* + * Some SoC core reset also reset PHY, more delay needed + * to make sure the reset process is done. + */ + usleep_range(1000, 2000); +} + +static void intel_pcie_device_rst_assert(struct intel_pcie_port *lpp) +{ + gpiod_set_value_cansleep(lpp->reset_gpio, 1); +} + +static void intel_pcie_device_rst_deassert(struct intel_pcie_port *lpp) +{ + msleep(lpp->rst_intrvl); + gpiod_set_value_cansleep(lpp->reset_gpio, 0); +} + +static int intel_pcie_app_logic_setup(struct intel_pcie_port *lpp) +{ + intel_pcie_device_rst_deassert(lpp); + intel_pcie_ltssm_enable(lpp); + + return dw_pcie_wait_for_link(&lpp->pci); +} + +static void intel_pcie_core_irq_disable(struct intel_pcie_port *lpp) +{ + pcie_app_wr(lpp, PCIE_APP_IRNEN, 0); + pcie_app_wr(lpp, PCIE_APP_IRNCR, PCIE_APP_IRN_INT); +} + +static int intel_pcie_get_resources(struct platform_device *pdev) +{ + struct intel_pcie_port *lpp = platform_get_drvdata(pdev); + struct dw_pcie *pci = &lpp->pci; + struct device *dev = pci->dev; + struct resource *res; + int ret; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + pci->dbi_base = devm_ioremap_resource(dev, res); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + lpp->core_clk = devm_clk_get(dev, NULL); + if (IS_ERR(lpp->core_clk)) { + ret = PTR_ERR(lpp->core_clk); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get clks: %d\n", ret); + return ret; + } + + lpp->core_rst = devm_reset_control_get(dev, NULL); + if (IS_ERR(lpp->core_rst)) { + ret = PTR_ERR(lpp->core_rst); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get resets: %d\n", ret); + return ret; + } + + ret = device_property_match_string(dev, "device_type", "pci"); + if (ret) { + dev_err(dev, "Failed to find pci device type: %d\n", ret); + return ret; + } + + ret = device_property_read_u32(dev, "reset-assert-ms", + &lpp->rst_intrvl); + if (ret) + lpp->rst_intrvl = RESET_INTERVAL_MS; + + ret = of_pci_get_max_link_speed(dev->of_node); + lpp->link_gen = ret < 0 ? 0 : ret; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "app"); + lpp->app_base = devm_ioremap_resource(dev, res); + if (IS_ERR(lpp->app_base)) + return PTR_ERR(lpp->app_base); + + lpp->phy = devm_phy_get(dev, "pcie"); + if (IS_ERR(lpp->phy)) { + ret = PTR_ERR(lpp->phy); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Couldn't get pcie-phy: %d\n", ret); + return ret; + } + + return 0; +} + +static void intel_pcie_deinit_phy(struct intel_pcie_port *lpp) +{ + phy_exit(lpp->phy); +} + +static int intel_pcie_wait_l2(struct intel_pcie_port *lpp) +{ + u32 value; + int ret; + + if (pcie_link_speed[lpp->max_speed] < PCIE_SPEED_8_0GT) + return 0; + + /* Send PME_TURN_OFF message */ + pcie_app_wr_mask(lpp, PCIE_APP_MSG_CR, PCIE_APP_MSG_XMT_PM_TURNOFF, + PCIE_APP_MSG_XMT_PM_TURNOFF); + + /* Read PMC status and wait for falling into L2 link state */ + ret = readl_poll_timeout(lpp->app_base + PCIE_APP_PMC, value, + value & PCIE_APP_PMC_IN_L2, 20, + jiffies_to_usecs(5 * HZ)); + if (ret) + dev_err(lpp->pci.dev, "PCIe link enter L2 timeout!\n"); + + return ret; +} + +static void intel_pcie_turn_off(struct intel_pcie_port *lpp) +{ + if (dw_pcie_link_up(&lpp->pci)) + intel_pcie_wait_l2(lpp); + + /* Put endpoint device in reset state */ + intel_pcie_device_rst_assert(lpp); + pcie_rc_cfg_wr_mask(lpp, PCI_COMMAND, PCI_COMMAND_MEMORY, 0); +} + +static int intel_pcie_host_setup(struct intel_pcie_port *lpp) +{ + struct device *dev = lpp->pci.dev; + int ret; + + intel_pcie_core_rst_assert(lpp); + intel_pcie_device_rst_assert(lpp); + + ret = phy_init(lpp->phy); + if (ret) + return ret; + + intel_pcie_core_rst_deassert(lpp); + + ret = clk_prepare_enable(lpp->core_clk); + if (ret) { + dev_err(lpp->pci.dev, "Core clock enable failed: %d\n", ret); + goto clk_err; + } + + if (!lpp->pcie_cap_ofst) { + ret = dw_pcie_find_capability(&lpp->pci, PCI_CAP_ID_EXP); + if (!ret) { + ret = -ENXIO; + dev_err(dev, "Invalid PCIe capability offset\n"); + goto app_init_err; + } + + lpp->pcie_cap_ofst = ret; + } + + intel_pcie_rc_setup(lpp); + ret = intel_pcie_app_logic_setup(lpp); + if (ret) + goto app_init_err; + + /* Enable integrated interrupts */ + pcie_app_wr_mask(lpp, PCIE_APP_IRNEN, PCIE_APP_IRN_INT, + PCIE_APP_IRN_INT); + + return 0; + +app_init_err: + clk_disable_unprepare(lpp->core_clk); +clk_err: + intel_pcie_core_rst_assert(lpp); + intel_pcie_deinit_phy(lpp); + + return ret; +} + +static void __intel_pcie_remove(struct intel_pcie_port *lpp) +{ + intel_pcie_core_irq_disable(lpp); + intel_pcie_turn_off(lpp); + clk_disable_unprepare(lpp->core_clk); + intel_pcie_core_rst_assert(lpp); + intel_pcie_deinit_phy(lpp); +} + +static int intel_pcie_remove(struct platform_device *pdev) +{ + struct intel_pcie_port *lpp = platform_get_drvdata(pdev); + struct pcie_port *pp = &lpp->pci.pp; + + dw_pcie_host_deinit(pp); + __intel_pcie_remove(lpp); + + return 0; +} + +static int __maybe_unused intel_pcie_suspend_noirq(struct device *dev) +{ + struct intel_pcie_port *lpp = dev_get_drvdata(dev); + int ret; + + intel_pcie_core_irq_disable(lpp); + ret = intel_pcie_wait_l2(lpp); + if (ret) + return ret; + + intel_pcie_deinit_phy(lpp); + clk_disable_unprepare(lpp->core_clk); + return ret; +} + +static int __maybe_unused intel_pcie_resume_noirq(struct device *dev) +{ + struct intel_pcie_port *lpp = dev_get_drvdata(dev); + + return intel_pcie_host_setup(lpp); +} + +static int intel_pcie_rc_init(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct intel_pcie_port *lpp = dev_get_drvdata(pci->dev); + + return intel_pcie_host_setup(lpp); +} + +/* + * Dummy function so that DW core doesn't configure MSI + */ +static int intel_pcie_msi_init(struct pcie_port *pp) +{ + return 0; +} + +u64 intel_pcie_cpu_addr(struct dw_pcie *pcie, u64 cpu_addr) +{ + return cpu_addr + BUS_IATU_OFFSET; +} + +static const struct dw_pcie_ops intel_pcie_ops = { + .cpu_addr_fixup = intel_pcie_cpu_addr, +}; + +static const struct dw_pcie_host_ops intel_pcie_dw_ops = { + .host_init = intel_pcie_rc_init, + .msi_host_init = intel_pcie_msi_init, +}; + +static const struct intel_pcie_soc pcie_data = { + .pcie_ver = 0x520A, + .pcie_atu_offset = 0xC0000, + .num_viewport = 3, +}; + +static int intel_pcie_probe(struct platform_device *pdev) +{ + const struct intel_pcie_soc *data; + struct device *dev = &pdev->dev; + struct intel_pcie_port *lpp; + struct pcie_port *pp; + struct dw_pcie *pci; + int ret; + + lpp = devm_kzalloc(dev, sizeof(*lpp), GFP_KERNEL); + if (!lpp) + return -ENOMEM; + + platform_set_drvdata(pdev, lpp); + pci = &lpp->pci; + pci->dev = dev; + pp = &pci->pp; + + ret = intel_pcie_get_resources(pdev); + if (ret) + return ret; + + ret = intel_pcie_ep_rst_init(lpp); + if (ret) + return ret; + + data = device_get_match_data(dev); + if (!data) + return -ENODEV; + + pci->ops = &intel_pcie_ops; + pci->version = data->pcie_ver; + pci->atu_base = pci->dbi_base + data->pcie_atu_offset; + pp->ops = &intel_pcie_dw_ops; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "Cannot initialize host\n"); + return ret; + } + + /* + * Intel PCIe doesn't configure IO region, so set viewport + * to not perform IO region access. + */ + pci->num_viewport = data->num_viewport; + + return 0; +} + +static const struct dev_pm_ops intel_pcie_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(intel_pcie_suspend_noirq, + intel_pcie_resume_noirq) +}; + +static const struct of_device_id of_intel_pcie_match[] = { + { .compatible = "intel,lgm-pcie", .data = &pcie_data }, + {} +}; + +static struct platform_driver intel_pcie_driver = { + .probe = intel_pcie_probe, + .remove = intel_pcie_remove, + .driver = { + .name = "intel-gw-pcie", + .of_match_table = of_intel_pcie_match, + .pm = &intel_pcie_pm_ops, + }, +}; +builtin_platform_driver(intel_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 7e581748ee9f..5ea527a6bd9f 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -54,6 +54,7 @@ #define PCIE20_PARF_LTSSM 0x1B0 #define PCIE20_PARF_SID_OFFSET 0x234 #define PCIE20_PARF_BDF_TRANSLATE_CFG 0x24C +#define PCIE20_PARF_DEVICE_TYPE 0x1000 #define PCIE20_ELBI_SYS_CTRL 0x04 #define PCIE20_ELBI_SYS_CTRL_LT_ENABLE BIT(0) @@ -80,6 +81,8 @@ #define PCIE20_v3_PARF_SLV_ADDR_SPACE_SIZE 0x358 #define SLV_ADDR_SPACE_SZ 0x10000000 +#define DEVICE_TYPE_RC 0x4 + #define QCOM_PCIE_2_1_0_MAX_SUPPLY 3 struct qcom_pcie_resources_2_1_0 { struct clk *iface_clk; @@ -139,12 +142,20 @@ struct qcom_pcie_resources_2_3_3 { struct reset_control *rst[7]; }; +struct qcom_pcie_resources_2_7_0 { + struct clk_bulk_data clks[6]; + struct regulator_bulk_data supplies[2]; + struct reset_control *pci_reset; + struct clk *pipe_clk; +}; + union qcom_pcie_resources { struct qcom_pcie_resources_1_0_0 v1_0_0; struct qcom_pcie_resources_2_1_0 v2_1_0; struct qcom_pcie_resources_2_3_2 v2_3_2; struct qcom_pcie_resources_2_3_3 v2_3_3; struct qcom_pcie_resources_2_4_0 v2_4_0; + struct qcom_pcie_resources_2_7_0 v2_7_0; }; struct qcom_pcie; @@ -1068,6 +1079,134 @@ err_clk_iface: return ret; } +static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + res->pci_reset = devm_reset_control_get_exclusive(dev, "pci"); + if (IS_ERR(res->pci_reset)) + return PTR_ERR(res->pci_reset); + + res->supplies[0].supply = "vdda"; + res->supplies[1].supply = "vddpe-3v3"; + ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies), + res->supplies); + if (ret) + return ret; + + res->clks[0].id = "aux"; + res->clks[1].id = "cfg"; + res->clks[2].id = "bus_master"; + res->clks[3].id = "bus_slave"; + res->clks[4].id = "slave_q2a"; + res->clks[5].id = "tbu"; + + ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks); + if (ret < 0) + return ret; + + res->pipe_clk = devm_clk_get(dev, "pipe"); + return PTR_ERR_OR_ZERO(res->pipe_clk); +} + +static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + u32 val; + int ret; + + ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); + if (ret < 0) { + dev_err(dev, "cannot enable regulators\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks); + if (ret < 0) + goto err_disable_regulators; + + ret = reset_control_assert(res->pci_reset); + if (ret < 0) { + dev_err(dev, "cannot deassert pci reset\n"); + goto err_disable_clocks; + } + + usleep_range(1000, 1500); + + ret = reset_control_deassert(res->pci_reset); + if (ret < 0) { + dev_err(dev, "cannot deassert pci reset\n"); + goto err_disable_clocks; + } + + ret = clk_prepare_enable(res->pipe_clk); + if (ret) { + dev_err(dev, "cannot prepare/enable pipe clock\n"); + goto err_disable_clocks; + } + + /* configure PCIe to RC mode */ + writel(DEVICE_TYPE_RC, pcie->parf + PCIE20_PARF_DEVICE_TYPE); + + /* enable PCIe clocks and resets */ + val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL); + val &= ~BIT(0); + writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL); + + /* change DBI base address */ + writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR); + + /* MAC PHY_POWERDOWN MUX DISABLE */ + val = readl(pcie->parf + PCIE20_PARF_SYS_CTRL); + val &= ~BIT(29); + writel(val, pcie->parf + PCIE20_PARF_SYS_CTRL); + + val = readl(pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL); + val |= BIT(4); + writel(val, pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL); + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + val = readl(pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT); + val |= BIT(31); + writel(val, pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT); + } + + return 0; +err_disable_clocks: + clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks); +err_disable_regulators: + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); + + return ret; +} + +static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + + clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks); + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); +} + +static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + + return clk_prepare_enable(res->pipe_clk); +} + +static void qcom_pcie_post_deinit_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + + clk_disable_unprepare(res->pipe_clk); +} + static int qcom_pcie_link_up(struct dw_pcie *pci) { u16 val = readw(pci->dbi_base + PCIE20_CAP + PCI_EXP_LNKSTA); @@ -1167,6 +1306,16 @@ static const struct qcom_pcie_ops ops_2_3_3 = { .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, }; +/* Qcom IP rev.: 2.7.0 Synopsys IP rev.: 4.30a */ +static const struct qcom_pcie_ops ops_2_7_0 = { + .get_resources = qcom_pcie_get_resources_2_7_0, + .init = qcom_pcie_init_2_7_0, + .deinit = qcom_pcie_deinit_2_7_0, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, + .post_init = qcom_pcie_post_init_2_7_0, + .post_deinit = qcom_pcie_post_deinit_2_7_0, +}; + static const struct dw_pcie_ops dw_pcie_ops = { .link_up = qcom_pcie_link_up, }; @@ -1282,6 +1431,7 @@ static const struct of_device_id qcom_pcie_match[] = { { .compatible = "qcom,pcie-ipq8074", .data = &ops_2_3_3 }, { .compatible = "qcom,pcie-ipq4019", .data = &ops_2_4_0 }, { .compatible = "qcom,pcie-qcs404", .data = &ops_2_4_0 }, + { .compatible = "qcom,pcie-sdm845", .data = &ops_2_7_0 }, { } }; diff --git a/drivers/pci/controller/dwc/pcie-uniphier.c b/drivers/pci/controller/dwc/pcie-uniphier.c index 8fd7badd59c2..a5401a0b1e58 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier.c +++ b/drivers/pci/controller/dwc/pcie-uniphier.c @@ -9,11 +9,11 @@ #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/delay.h> +#include <linux/init.h> #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/irqchip/chained_irq.h> #include <linux/irqdomain.h> -#include <linux/module.h> #include <linux/of_irq.h> #include <linux/pci.h> #include <linux/phy/phy.h> @@ -171,12 +171,6 @@ static void uniphier_pcie_irq_enable(struct uniphier_pcie_priv *priv) writel(PCL_RCV_INTX_ALL_ENABLE, priv->base + PCL_RCV_INTX); } -static void uniphier_pcie_irq_disable(struct uniphier_pcie_priv *priv) -{ - writel(0, priv->base + PCL_RCV_INT); - writel(0, priv->base + PCL_RCV_INTX); -} - static void uniphier_pcie_irq_ack(struct irq_data *d) { struct pcie_port *pp = irq_data_get_irq_chip_data(d); @@ -397,14 +391,6 @@ out_clk_disable: return ret; } -static void uniphier_pcie_host_disable(struct uniphier_pcie_priv *priv) -{ - uniphier_pcie_irq_disable(priv); - phy_exit(priv->phy); - reset_control_assert(priv->rst); - clk_disable_unprepare(priv->clk); -} - static const struct dw_pcie_ops dw_pcie_ops = { .start_link = uniphier_pcie_establish_link, .stop_link = uniphier_pcie_stop_link, @@ -456,31 +442,16 @@ static int uniphier_pcie_probe(struct platform_device *pdev) return uniphier_add_pcie_port(priv, pdev); } -static int uniphier_pcie_remove(struct platform_device *pdev) -{ - struct uniphier_pcie_priv *priv = platform_get_drvdata(pdev); - - uniphier_pcie_host_disable(priv); - - return 0; -} - static const struct of_device_id uniphier_pcie_match[] = { { .compatible = "socionext,uniphier-pcie", }, { /* sentinel */ }, }; -MODULE_DEVICE_TABLE(of, uniphier_pcie_match); static struct platform_driver uniphier_pcie_driver = { .probe = uniphier_pcie_probe, - .remove = uniphier_pcie_remove, .driver = { .name = "uniphier-pcie", .of_match_table = uniphier_pcie_match, }, }; builtin_platform_driver(uniphier_pcie_driver); - -MODULE_AUTHOR("Kunihiko Hayashi <hayashi.kunihiko@socionext.com>"); -MODULE_DESCRIPTION("UniPhier PCIe host controller driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index 673a1725ef38..0e03cef72840 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -1406,7 +1406,7 @@ static struct phy *devm_of_phy_optional_get_index(struct device *dev, phy = devm_of_phy_get(dev, np, name); kfree(name); - if (IS_ERR(phy) && PTR_ERR(phy) == -ENODEV) + if (PTR_ERR(phy) == -ENODEV) phy = NULL; return phy; @@ -2499,7 +2499,6 @@ static const struct tegra_pcie_soc tegra20_pcie = { .num_ports = 2, .ports = tegra20_pcie_ports, .msi_base_shift = 0, - .afi_pex2_ctrl = 0x128, .pads_pll_ctl = PADS_PLL_CTL_TEGRA20, .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10, .pads_refclk_cfg0 = 0xfa5cfa5c, @@ -2528,6 +2527,7 @@ static const struct tegra_pcie_soc tegra30_pcie = { .num_ports = 3, .ports = tegra30_pcie_ports, .msi_base_shift = 8, + .afi_pex2_ctrl = 0x128, .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, .pads_refclk_cfg0 = 0xfa5cfa5c, @@ -2798,7 +2798,7 @@ static int tegra_pcie_probe(struct platform_device *pdev) pm_runtime_enable(pcie->dev); err = pm_runtime_get_sync(pcie->dev); - if (err) { + if (err < 0) { dev_err(dev, "fail to enable pcie controller: %d\n", err); goto teardown_msi; } diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c new file mode 100644 index 000000000000..d20aabc26273 --- /dev/null +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -0,0 +1,1015 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (C) 2009 - 2019 Broadcom */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/compiler.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/ioport.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/log2.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/printk.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/types.h> + +#include "../pci.h" + +/* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */ +#define BRCM_PCIE_CAP_REGS 0x00ac + +/* Broadcom STB PCIe Register Offsets */ +#define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1 0x0188 +#define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK 0xc +#define PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN 0x0 + +#define PCIE_RC_CFG_PRIV1_ID_VAL3 0x043c +#define PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK 0xffffff + +#define PCIE_RC_DL_MDIO_ADDR 0x1100 +#define PCIE_RC_DL_MDIO_WR_DATA 0x1104 +#define PCIE_RC_DL_MDIO_RD_DATA 0x1108 + +#define PCIE_MISC_MISC_CTRL 0x4008 +#define PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK 0x1000 +#define PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK 0x2000 +#define PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK 0x300000 +#define PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_128 0x0 +#define PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK 0xf8000000 + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO 0x400c +#define PCIE_MEM_WIN0_LO(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 4) + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI 0x4010 +#define PCIE_MEM_WIN0_HI(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 4) + +#define PCIE_MISC_RC_BAR1_CONFIG_LO 0x402c +#define PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK 0x1f + +#define PCIE_MISC_RC_BAR2_CONFIG_LO 0x4034 +#define PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK 0x1f +#define PCIE_MISC_RC_BAR2_CONFIG_HI 0x4038 + +#define PCIE_MISC_RC_BAR3_CONFIG_LO 0x403c +#define PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK 0x1f + +#define PCIE_MISC_MSI_BAR_CONFIG_LO 0x4044 +#define PCIE_MISC_MSI_BAR_CONFIG_HI 0x4048 + +#define PCIE_MISC_MSI_DATA_CONFIG 0x404c +#define PCIE_MISC_MSI_DATA_CONFIG_VAL 0xffe06540 + +#define PCIE_MISC_PCIE_CTRL 0x4064 +#define PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK 0x1 + +#define PCIE_MISC_PCIE_STATUS 0x4068 +#define PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK 0x80 +#define PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK 0x20 +#define PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK 0x10 +#define PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK 0x40 + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT 0x4070 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK 0xfff00000 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK 0xfff0 +#define PCIE_MEM_WIN0_BASE_LIMIT(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4) + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI 0x4080 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK 0xff +#define PCIE_MEM_WIN0_BASE_HI(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8) + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI 0x4084 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK 0xff +#define PCIE_MEM_WIN0_LIMIT_HI(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8) + +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG 0x4204 +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK 0x2 +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x08000000 + +#define PCIE_MSI_INTR2_STATUS 0x4500 +#define PCIE_MSI_INTR2_CLR 0x4508 +#define PCIE_MSI_INTR2_MASK_SET 0x4510 +#define PCIE_MSI_INTR2_MASK_CLR 0x4514 + +#define PCIE_EXT_CFG_DATA 0x8000 + +#define PCIE_EXT_CFG_INDEX 0x9000 +#define PCIE_EXT_BUSNUM_SHIFT 20 +#define PCIE_EXT_SLOT_SHIFT 15 +#define PCIE_EXT_FUNC_SHIFT 12 + +#define PCIE_RGR1_SW_INIT_1 0x9210 +#define PCIE_RGR1_SW_INIT_1_PERST_MASK 0x1 +#define PCIE_RGR1_SW_INIT_1_INIT_MASK 0x2 + +/* PCIe parameters */ +#define BRCM_NUM_PCIE_OUT_WINS 0x4 +#define BRCM_INT_PCI_MSI_NR 32 + +/* MSI target adresses */ +#define BRCM_MSI_TARGET_ADDR_LT_4GB 0x0fffffffcULL +#define BRCM_MSI_TARGET_ADDR_GT_4GB 0xffffffffcULL + +/* MDIO registers */ +#define MDIO_PORT0 0x0 +#define MDIO_DATA_MASK 0x7fffffff +#define MDIO_PORT_MASK 0xf0000 +#define MDIO_REGAD_MASK 0xffff +#define MDIO_CMD_MASK 0xfff00000 +#define MDIO_CMD_READ 0x1 +#define MDIO_CMD_WRITE 0x0 +#define MDIO_DATA_DONE_MASK 0x80000000 +#define MDIO_RD_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 1 : 0) +#define MDIO_WT_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 0 : 1) +#define SSC_REGS_ADDR 0x1100 +#define SET_ADDR_OFFSET 0x1f +#define SSC_CNTL_OFFSET 0x2 +#define SSC_CNTL_OVRD_EN_MASK 0x8000 +#define SSC_CNTL_OVRD_VAL_MASK 0x4000 +#define SSC_STATUS_OFFSET 0x1 +#define SSC_STATUS_SSC_MASK 0x400 +#define SSC_STATUS_PLL_LOCK_MASK 0x800 + +struct brcm_msi { + struct device *dev; + void __iomem *base; + struct device_node *np; + struct irq_domain *msi_domain; + struct irq_domain *inner_domain; + struct mutex lock; /* guards the alloc/free operations */ + u64 target_addr; + int irq; + /* used indicates which MSI interrupts have been alloc'd */ + unsigned long used; +}; + +/* Internal PCIe Host Controller Information.*/ +struct brcm_pcie { + struct device *dev; + void __iomem *base; + struct clk *clk; + struct pci_bus *root_bus; + struct device_node *np; + bool ssc; + int gen; + u64 msi_target_addr; + struct brcm_msi *msi; +}; + +/* + * This is to convert the size of the inbound "BAR" region to the + * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE + */ +static int brcm_pcie_encode_ibar_size(u64 size) +{ + int log2_in = ilog2(size); + + if (log2_in >= 12 && log2_in <= 15) + /* Covers 4KB to 32KB (inclusive) */ + return (log2_in - 12) + 0x1c; + else if (log2_in >= 16 && log2_in <= 35) + /* Covers 64KB to 32GB, (inclusive) */ + return log2_in - 15; + /* Something is awry so disable */ + return 0; +} + +static u32 brcm_pcie_mdio_form_pkt(int port, int regad, int cmd) +{ + u32 pkt = 0; + + pkt |= FIELD_PREP(MDIO_PORT_MASK, port); + pkt |= FIELD_PREP(MDIO_REGAD_MASK, regad); + pkt |= FIELD_PREP(MDIO_CMD_MASK, cmd); + + return pkt; +} + +/* negative return value indicates error */ +static int brcm_pcie_mdio_read(void __iomem *base, u8 port, u8 regad, u32 *val) +{ + int tries; + u32 data; + + writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ), + base + PCIE_RC_DL_MDIO_ADDR); + readl(base + PCIE_RC_DL_MDIO_ADDR); + + data = readl(base + PCIE_RC_DL_MDIO_RD_DATA); + for (tries = 0; !MDIO_RD_DONE(data) && tries < 10; tries++) { + udelay(10); + data = readl(base + PCIE_RC_DL_MDIO_RD_DATA); + } + + *val = FIELD_GET(MDIO_DATA_MASK, data); + return MDIO_RD_DONE(data) ? 0 : -EIO; +} + +/* negative return value indicates error */ +static int brcm_pcie_mdio_write(void __iomem *base, u8 port, + u8 regad, u16 wrdata) +{ + int tries; + u32 data; + + writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE), + base + PCIE_RC_DL_MDIO_ADDR); + readl(base + PCIE_RC_DL_MDIO_ADDR); + writel(MDIO_DATA_DONE_MASK | wrdata, base + PCIE_RC_DL_MDIO_WR_DATA); + + data = readl(base + PCIE_RC_DL_MDIO_WR_DATA); + for (tries = 0; !MDIO_WT_DONE(data) && tries < 10; tries++) { + udelay(10); + data = readl(base + PCIE_RC_DL_MDIO_WR_DATA); + } + + return MDIO_WT_DONE(data) ? 0 : -EIO; +} + +/* + * Configures device for Spread Spectrum Clocking (SSC) mode; a negative + * return value indicates error. + */ +static int brcm_pcie_set_ssc(struct brcm_pcie *pcie) +{ + int pll, ssc; + int ret; + u32 tmp; + + ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, + SSC_REGS_ADDR); + if (ret < 0) + return ret; + + ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, + SSC_CNTL_OFFSET, &tmp); + if (ret < 0) + return ret; + + u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_EN_MASK); + u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_VAL_MASK); + ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, + SSC_CNTL_OFFSET, tmp); + if (ret < 0) + return ret; + + usleep_range(1000, 2000); + ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, + SSC_STATUS_OFFSET, &tmp); + if (ret < 0) + return ret; + + ssc = FIELD_GET(SSC_STATUS_SSC_MASK, tmp); + pll = FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp); + + return ssc && pll ? 0 : -EIO; +} + +/* Limits operation to a specific generation (1, 2, or 3) */ +static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen) +{ + u16 lnkctl2 = readw(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2); + u32 lnkcap = readl(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP); + + lnkcap = (lnkcap & ~PCI_EXP_LNKCAP_SLS) | gen; + writel(lnkcap, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP); + + lnkctl2 = (lnkctl2 & ~0xf) | gen; + writew(lnkctl2, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2); +} + +static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie, + unsigned int win, u64 cpu_addr, + u64 pcie_addr, u64 size) +{ + u32 cpu_addr_mb_high, limit_addr_mb_high; + phys_addr_t cpu_addr_mb, limit_addr_mb; + int high_addr_shift; + u32 tmp; + + /* Set the base of the pcie_addr window */ + writel(lower_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_LO(win)); + writel(upper_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_HI(win)); + + /* Write the addr base & limit lower bits (in MBs) */ + cpu_addr_mb = cpu_addr / SZ_1M; + limit_addr_mb = (cpu_addr + size - 1) / SZ_1M; + + tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)); + u32p_replace_bits(&tmp, cpu_addr_mb, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); + u32p_replace_bits(&tmp, limit_addr_mb, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK); + writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)); + + /* Write the cpu & limit addr upper bits */ + high_addr_shift = + HWEIGHT32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); + + cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift; + tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_HI(win)); + u32p_replace_bits(&tmp, cpu_addr_mb_high, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK); + writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_HI(win)); + + limit_addr_mb_high = limit_addr_mb >> high_addr_shift; + tmp = readl(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); + u32p_replace_bits(&tmp, limit_addr_mb_high, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK); + writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); +} + +static struct irq_chip brcm_msi_irq_chip = { + .name = "BRCM STB PCIe MSI", + .irq_ack = irq_chip_ack_parent, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, +}; + +static struct msi_domain_info brcm_msi_domain_info = { + /* Multi MSI is supported by the controller, but not by this driver */ + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS), + .chip = &brcm_msi_irq_chip, +}; + +static void brcm_pcie_msi_isr(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long status, virq; + struct brcm_msi *msi; + struct device *dev; + u32 bit; + + chained_irq_enter(chip, desc); + msi = irq_desc_get_handler_data(desc); + dev = msi->dev; + + status = readl(msi->base + PCIE_MSI_INTR2_STATUS); + for_each_set_bit(bit, &status, BRCM_INT_PCI_MSI_NR) { + virq = irq_find_mapping(msi->inner_domain, bit); + if (virq) + generic_handle_irq(virq); + else + dev_dbg(dev, "unexpected MSI\n"); + } + + chained_irq_exit(chip, desc); +} + +static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct brcm_msi *msi = irq_data_get_irq_chip_data(data); + + msg->address_lo = lower_32_bits(msi->target_addr); + msg->address_hi = upper_32_bits(msi->target_addr); + msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL) | data->hwirq; +} + +static int brcm_msi_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) +{ + return -EINVAL; +} + +static void brcm_msi_ack_irq(struct irq_data *data) +{ + struct brcm_msi *msi = irq_data_get_irq_chip_data(data); + + writel(1 << data->hwirq, msi->base + PCIE_MSI_INTR2_CLR); +} + + +static struct irq_chip brcm_msi_bottom_irq_chip = { + .name = "BRCM STB MSI", + .irq_compose_msi_msg = brcm_msi_compose_msi_msg, + .irq_set_affinity = brcm_msi_set_affinity, + .irq_ack = brcm_msi_ack_irq, +}; + +static int brcm_msi_alloc(struct brcm_msi *msi) +{ + int hwirq; + + mutex_lock(&msi->lock); + hwirq = bitmap_find_free_region(&msi->used, BRCM_INT_PCI_MSI_NR, 0); + mutex_unlock(&msi->lock); + + return hwirq; +} + +static void brcm_msi_free(struct brcm_msi *msi, unsigned long hwirq) +{ + mutex_lock(&msi->lock); + bitmap_release_region(&msi->used, hwirq, 0); + mutex_unlock(&msi->lock); +} + +static int brcm_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct brcm_msi *msi = domain->host_data; + int hwirq; + + hwirq = brcm_msi_alloc(msi); + + if (hwirq < 0) + return hwirq; + + irq_domain_set_info(domain, virq, (irq_hw_number_t)hwirq, + &brcm_msi_bottom_irq_chip, domain->host_data, + handle_edge_irq, NULL, NULL); + return 0; +} + +static void brcm_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct brcm_msi *msi = irq_data_get_irq_chip_data(d); + + brcm_msi_free(msi, d->hwirq); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = brcm_irq_domain_alloc, + .free = brcm_irq_domain_free, +}; + +static int brcm_allocate_domains(struct brcm_msi *msi) +{ + struct fwnode_handle *fwnode = of_node_to_fwnode(msi->np); + struct device *dev = msi->dev; + + msi->inner_domain = irq_domain_add_linear(NULL, BRCM_INT_PCI_MSI_NR, + &msi_domain_ops, msi); + if (!msi->inner_domain) { + dev_err(dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + msi->msi_domain = pci_msi_create_irq_domain(fwnode, + &brcm_msi_domain_info, + msi->inner_domain); + if (!msi->msi_domain) { + dev_err(dev, "failed to create MSI domain\n"); + irq_domain_remove(msi->inner_domain); + return -ENOMEM; + } + + return 0; +} + +static void brcm_free_domains(struct brcm_msi *msi) +{ + irq_domain_remove(msi->msi_domain); + irq_domain_remove(msi->inner_domain); +} + +static void brcm_msi_remove(struct brcm_pcie *pcie) +{ + struct brcm_msi *msi = pcie->msi; + + if (!msi) + return; + irq_set_chained_handler(msi->irq, NULL); + irq_set_handler_data(msi->irq, NULL); + brcm_free_domains(msi); +} + +static void brcm_msi_set_regs(struct brcm_msi *msi) +{ + writel(0xffffffff, msi->base + PCIE_MSI_INTR2_MASK_CLR); + + /* + * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI + * enable, which we set to 1. + */ + writel(lower_32_bits(msi->target_addr) | 0x1, + msi->base + PCIE_MISC_MSI_BAR_CONFIG_LO); + writel(upper_32_bits(msi->target_addr), + msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI); + + writel(PCIE_MISC_MSI_DATA_CONFIG_VAL, + msi->base + PCIE_MISC_MSI_DATA_CONFIG); +} + +static int brcm_pcie_enable_msi(struct brcm_pcie *pcie) +{ + struct brcm_msi *msi; + int irq, ret; + struct device *dev = pcie->dev; + + irq = irq_of_parse_and_map(dev->of_node, 1); + if (irq <= 0) { + dev_err(dev, "cannot map MSI interrupt\n"); + return -ENODEV; + } + + msi = devm_kzalloc(dev, sizeof(struct brcm_msi), GFP_KERNEL); + if (!msi) + return -ENOMEM; + + mutex_init(&msi->lock); + msi->dev = dev; + msi->base = pcie->base; + msi->np = pcie->np; + msi->target_addr = pcie->msi_target_addr; + msi->irq = irq; + + ret = brcm_allocate_domains(msi); + if (ret) + return ret; + + irq_set_chained_handler_and_data(msi->irq, brcm_pcie_msi_isr, msi); + + brcm_msi_set_regs(msi); + pcie->msi = msi; + + return 0; +} + +/* The controller is capable of serving in both RC and EP roles */ +static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie) +{ + void __iomem *base = pcie->base; + u32 val = readl(base + PCIE_MISC_PCIE_STATUS); + + return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val); +} + +static bool brcm_pcie_link_up(struct brcm_pcie *pcie) +{ + u32 val = readl(pcie->base + PCIE_MISC_PCIE_STATUS); + u32 dla = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val); + u32 plu = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val); + + return dla && plu; +} + +/* Configuration space read/write support */ +static inline int brcm_pcie_cfg_index(int busnr, int devfn, int reg) +{ + return ((PCI_SLOT(devfn) & 0x1f) << PCIE_EXT_SLOT_SHIFT) + | ((PCI_FUNC(devfn) & 0x07) << PCIE_EXT_FUNC_SHIFT) + | (busnr << PCIE_EXT_BUSNUM_SHIFT) + | (reg & ~3); +} + +static void __iomem *brcm_pcie_map_conf(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct brcm_pcie *pcie = bus->sysdata; + void __iomem *base = pcie->base; + int idx; + + /* Accesses to the RC go right to the RC registers if slot==0 */ + if (pci_is_root_bus(bus)) + return PCI_SLOT(devfn) ? NULL : base + where; + + /* For devices, write to the config space index register */ + idx = brcm_pcie_cfg_index(bus->number, devfn, 0); + writel(idx, pcie->base + PCIE_EXT_CFG_INDEX); + return base + PCIE_EXT_CFG_DATA + where; +} + +static struct pci_ops brcm_pcie_ops = { + .map_bus = brcm_pcie_map_conf, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static inline void brcm_pcie_bridge_sw_init_set(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp; + + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1); + u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_INIT_MASK); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1); +} + +static inline void brcm_pcie_perst_set(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp; + + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1); + u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1); +} + +static inline int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, + u64 *rc_bar2_size, + u64 *rc_bar2_offset) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct device *dev = pcie->dev; + struct resource_entry *entry; + + entry = resource_list_first_type(&bridge->dma_ranges, IORESOURCE_MEM); + if (!entry) + return -ENODEV; + + + /* + * The controller expects the inbound window offset to be calculated as + * the difference between PCIe's address space and CPU's. The offset + * provided by the firmware is calculated the opposite way, so we + * negate it. + */ + *rc_bar2_offset = -entry->offset; + *rc_bar2_size = 1ULL << fls64(entry->res->end - entry->res->start); + + /* + * We validate the inbound memory view even though we should trust + * whatever the device-tree provides. This is because of an HW issue on + * early Raspberry Pi 4's revisions (bcm2711). It turns out its + * firmware has to dynamically edit dma-ranges due to a bug on the + * PCIe controller integration, which prohibits any access above the + * lower 3GB of memory. Given this, we decided to keep the dma-ranges + * in check, avoiding hard to debug device-tree related issues in the + * future: + * + * The PCIe host controller by design must set the inbound viewport to + * be a contiguous arrangement of all of the system's memory. In + * addition, its size mut be a power of two. To further complicate + * matters, the viewport must start on a pcie-address that is aligned + * on a multiple of its size. If a portion of the viewport does not + * represent system memory -- e.g. 3GB of memory requires a 4GB + * viewport -- we can map the outbound memory in or after 3GB and even + * though the viewport will overlap the outbound memory the controller + * will know to send outbound memory downstream and everything else + * upstream. + * + * For example: + * + * - The best-case scenario, memory up to 3GB, is to place the inbound + * region in the first 4GB of pcie-space, as some legacy devices can + * only address 32bits. We would also like to put the MSI under 4GB + * as well, since some devices require a 32bit MSI target address. + * + * - If the system memory is 4GB or larger we cannot start the inbound + * region at location 0 (since we have to allow some space for + * outbound memory @ 3GB). So instead it will start at the 1x + * multiple of its size + */ + if (!*rc_bar2_size || *rc_bar2_offset % *rc_bar2_size || + (*rc_bar2_offset < SZ_4G && *rc_bar2_offset > SZ_2G)) { + dev_err(dev, "Invalid rc_bar2_offset/size: size 0x%llx, off 0x%llx\n", + *rc_bar2_size, *rc_bar2_offset); + return -EINVAL; + } + + return 0; +} + +static int brcm_pcie_setup(struct brcm_pcie *pcie) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + u64 rc_bar2_offset, rc_bar2_size; + void __iomem *base = pcie->base; + struct device *dev = pcie->dev; + struct resource_entry *entry; + unsigned int scb_size_val; + bool ssc_good = false; + struct resource *res; + int num_out_wins = 0; + u16 nlw, cls, lnksta; + int i, ret; + u32 tmp; + + /* Reset the bridge */ + brcm_pcie_bridge_sw_init_set(pcie, 1); + + usleep_range(100, 200); + + /* Take the bridge out of reset */ + brcm_pcie_bridge_sw_init_set(pcie, 0); + + tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; + writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + /* Wait for SerDes to be stable */ + usleep_range(100, 200); + + /* Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN */ + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK); + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK); + u32p_replace_bits(&tmp, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_128, + PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK); + writel(tmp, base + PCIE_MISC_MISC_CTRL); + + ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, &rc_bar2_size, + &rc_bar2_offset); + if (ret) + return ret; + + tmp = lower_32_bits(rc_bar2_offset); + u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(rc_bar2_size), + PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK); + writel(tmp, base + PCIE_MISC_RC_BAR2_CONFIG_LO); + writel(upper_32_bits(rc_bar2_offset), + base + PCIE_MISC_RC_BAR2_CONFIG_HI); + + scb_size_val = rc_bar2_size ? + ilog2(rc_bar2_size) - 15 : 0xf; /* 0xf is 1GB */ + tmp = readl(base + PCIE_MISC_MISC_CTRL); + u32p_replace_bits(&tmp, scb_size_val, + PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK); + writel(tmp, base + PCIE_MISC_MISC_CTRL); + + /* + * We ideally want the MSI target address to be located in the 32bit + * addressable memory area. Some devices might depend on it. This is + * possible either when the inbound window is located above the lower + * 4GB or when the inbound area is smaller than 4GB (taking into + * account the rounding-up we're forced to perform). + */ + if (rc_bar2_offset >= SZ_4G || (rc_bar2_size + rc_bar2_offset) < SZ_4G) + pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB; + else + pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB; + + /* disable the PCIe->GISB memory window (RC_BAR1) */ + tmp = readl(base + PCIE_MISC_RC_BAR1_CONFIG_LO); + tmp &= ~PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK; + writel(tmp, base + PCIE_MISC_RC_BAR1_CONFIG_LO); + + /* disable the PCIe->SCB memory window (RC_BAR3) */ + tmp = readl(base + PCIE_MISC_RC_BAR3_CONFIG_LO); + tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK; + writel(tmp, base + PCIE_MISC_RC_BAR3_CONFIG_LO); + + /* Mask all interrupts since we are not handling any yet */ + writel(0xffffffff, pcie->base + PCIE_MSI_INTR2_MASK_SET); + + /* clear any interrupts we find on boot */ + writel(0xffffffff, pcie->base + PCIE_MSI_INTR2_CLR); + + if (pcie->gen) + brcm_pcie_set_gen(pcie, pcie->gen); + + /* Unassert the fundamental reset */ + brcm_pcie_perst_set(pcie, 0); + + /* + * Give the RC/EP time to wake up, before trying to configure RC. + * Intermittently check status for link-up, up to a total of 100ms. + */ + for (i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5) + msleep(5); + + if (!brcm_pcie_link_up(pcie)) { + dev_err(dev, "link down\n"); + return -ENODEV; + } + + if (!brcm_pcie_rc_mode(pcie)) { + dev_err(dev, "PCIe misconfigured; is in EP mode\n"); + return -EINVAL; + } + + resource_list_for_each_entry(entry, &bridge->windows) { + res = entry->res; + + if (resource_type(res) != IORESOURCE_MEM) + continue; + + if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) { + dev_err(pcie->dev, "too many outbound wins\n"); + return -EINVAL; + } + + brcm_pcie_set_outbound_win(pcie, num_out_wins, res->start, + res->start - entry->offset, + resource_size(res)); + num_out_wins++; + } + + /* + * For config space accesses on the RC, show the right class for + * a PCIe-PCIe bridge (the default setting is to be EP mode). + */ + tmp = readl(base + PCIE_RC_CFG_PRIV1_ID_VAL3); + u32p_replace_bits(&tmp, 0x060400, + PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK); + writel(tmp, base + PCIE_RC_CFG_PRIV1_ID_VAL3); + + if (pcie->ssc) { + ret = brcm_pcie_set_ssc(pcie); + if (ret == 0) + ssc_good = true; + else + dev_err(dev, "failed attempt to enter ssc mode\n"); + } + + lnksta = readw(base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKSTA); + cls = FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta); + nlw = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + dev_info(dev, "link up, %s x%u %s\n", + PCIE_SPEED2STR(cls + PCI_SPEED_133MHz_PCIX_533), + nlw, ssc_good ? "(SSC)" : "(!SSC)"); + + /* PCIe->SCB endian mode for BAR */ + tmp = readl(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); + u32p_replace_bits(&tmp, PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN, + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK); + writel(tmp, base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); + + /* + * Refclk from RC should be gated with CLKREQ# input when ASPM L0s,L1 + * is enabled => setting the CLKREQ_DEBUG_ENABLE field to 1. + */ + tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + tmp |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK; + writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + + return 0; +} + +/* L23 is a low-power PCIe link state */ +static void brcm_pcie_enter_l23(struct brcm_pcie *pcie) +{ + void __iomem *base = pcie->base; + int l23, i; + u32 tmp; + + /* Assert request for L23 */ + tmp = readl(base + PCIE_MISC_PCIE_CTRL); + u32p_replace_bits(&tmp, 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); + writel(tmp, base + PCIE_MISC_PCIE_CTRL); + + /* Wait up to 36 msec for L23 */ + tmp = readl(base + PCIE_MISC_PCIE_STATUS); + l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp); + for (i = 0; i < 15 && !l23; i++) { + usleep_range(2000, 2400); + tmp = readl(base + PCIE_MISC_PCIE_STATUS); + l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, + tmp); + } + + if (!l23) + dev_err(pcie->dev, "failed to enter low-power link state\n"); +} + +static void brcm_pcie_turn_off(struct brcm_pcie *pcie) +{ + void __iomem *base = pcie->base; + int tmp; + + if (brcm_pcie_link_up(pcie)) + brcm_pcie_enter_l23(pcie); + /* Assert fundamental reset */ + brcm_pcie_perst_set(pcie, 1); + + /* Deassert request for L23 in case it was asserted */ + tmp = readl(base + PCIE_MISC_PCIE_CTRL); + u32p_replace_bits(&tmp, 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); + writel(tmp, base + PCIE_MISC_PCIE_CTRL); + + /* Turn off SerDes */ + tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + u32p_replace_bits(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); + writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG); + + /* Shutdown PCIe bridge */ + brcm_pcie_bridge_sw_init_set(pcie, 1); +} + +static void __brcm_pcie_remove(struct brcm_pcie *pcie) +{ + brcm_msi_remove(pcie); + brcm_pcie_turn_off(pcie); + clk_disable_unprepare(pcie->clk); + clk_put(pcie->clk); +} + +static int brcm_pcie_remove(struct platform_device *pdev) +{ + struct brcm_pcie *pcie = platform_get_drvdata(pdev); + + pci_stop_root_bus(pcie->root_bus); + pci_remove_root_bus(pcie->root_bus); + __brcm_pcie_remove(pcie); + + return 0; +} + +static int brcm_pcie_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node, *msi_np; + struct pci_host_bridge *bridge; + struct brcm_pcie *pcie; + struct pci_bus *child; + struct resource *res; + int ret; + + bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->dev = &pdev->dev; + pcie->np = np; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + pcie->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + pcie->clk = devm_clk_get_optional(&pdev->dev, "sw_pcie"); + if (IS_ERR(pcie->clk)) + return PTR_ERR(pcie->clk); + + ret = of_pci_get_max_link_speed(np); + pcie->gen = (ret < 0) ? 0 : ret; + + pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc"); + + ret = pci_parse_request_of_pci_ranges(pcie->dev, &bridge->windows, + &bridge->dma_ranges, NULL); + if (ret) + return ret; + + ret = clk_prepare_enable(pcie->clk); + if (ret) { + dev_err(&pdev->dev, "could not enable clock\n"); + return ret; + } + + ret = brcm_pcie_setup(pcie); + if (ret) + goto fail; + + msi_np = of_parse_phandle(pcie->np, "msi-parent", 0); + if (pci_msi_enabled() && msi_np == pcie->np) { + ret = brcm_pcie_enable_msi(pcie); + if (ret) { + dev_err(pcie->dev, "probe of internal MSI failed"); + goto fail; + } + } + + bridge->dev.parent = &pdev->dev; + bridge->busnr = 0; + bridge->ops = &brcm_pcie_ops; + bridge->sysdata = pcie; + bridge->map_irq = of_irq_parse_and_map_pci; + bridge->swizzle_irq = pci_common_swizzle; + + ret = pci_scan_root_bus_bridge(bridge); + if (ret < 0) { + dev_err(pcie->dev, "Scanning root bridge failed\n"); + goto fail; + } + + pci_assign_unassigned_bus_resources(bridge->bus); + list_for_each_entry(child, &bridge->bus->children, node) + pcie_bus_configure_settings(child); + pci_bus_add_devices(bridge->bus); + platform_set_drvdata(pdev, pcie); + pcie->root_bus = bridge->bus; + + return 0; +fail: + __brcm_pcie_remove(pcie); + return ret; +} + +static const struct of_device_id brcm_pcie_match[] = { + { .compatible = "brcm,bcm2711-pcie" }, + {}, +}; +MODULE_DEVICE_TABLE(of, brcm_pcie_match); + +static struct platform_driver brcm_pcie_driver = { + .probe = brcm_pcie_probe, + .remove = brcm_pcie_remove, + .driver = { + .name = "brcm-pcie", + .of_match_table = brcm_pcie_match, + }, +}; +module_platform_driver(brcm_pcie_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Broadcom STB PCIe RC driver"); +MODULE_AUTHOR("Broadcom"); diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index 0a468c73bae3..8c7f875acf7f 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -1588,6 +1588,30 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_disable_msi_parsing); +static void quirk_paxc_bridge(struct pci_dev *pdev) +{ + /* + * The PCI config space is shared with the PAXC root port and the first + * Ethernet device. So, we need to workaround this by telling the PCI + * code that the bridge is not an Ethernet device. + */ + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + pdev->class = PCI_CLASS_BRIDGE_PCI << 8; + + /* + * MPSS is not being set properly (as it is currently 0). This is + * because that area of the PCI config space is hard coded to zero, and + * is not modifiable by firmware. Set this to 2 (e.g., 512 byte MPS) + * so that the MPS can be set to the real max value. + */ + pdev->pcie_mpss = 2; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge); + MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>"); MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index 212842263f55..dac91d60701d 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -98,9 +98,6 @@ struct vmd_dev { struct irq_domain *irq_domain; struct pci_bus *bus; u8 busn_start; - - struct dma_map_ops dma_ops; - struct dma_domain dma_domain; }; static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus) @@ -295,151 +292,6 @@ static struct msi_domain_info vmd_msi_domain_info = { .chip = &vmd_msi_controller, }; -/* - * VMD replaces the requester ID with its own. DMA mappings for devices in a - * VMD domain need to be mapped for the VMD, not the device requiring - * the mapping. - */ -static struct device *to_vmd_dev(struct device *dev) -{ - struct pci_dev *pdev = to_pci_dev(dev); - struct vmd_dev *vmd = vmd_from_bus(pdev->bus); - - return &vmd->dev->dev; -} - -static void *vmd_alloc(struct device *dev, size_t size, dma_addr_t *addr, - gfp_t flag, unsigned long attrs) -{ - return dma_alloc_attrs(to_vmd_dev(dev), size, addr, flag, attrs); -} - -static void vmd_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t addr, unsigned long attrs) -{ - return dma_free_attrs(to_vmd_dev(dev), size, vaddr, addr, attrs); -} - -static int vmd_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t addr, size_t size, - unsigned long attrs) -{ - return dma_mmap_attrs(to_vmd_dev(dev), vma, cpu_addr, addr, size, - attrs); -} - -static int vmd_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t addr, size_t size, - unsigned long attrs) -{ - return dma_get_sgtable_attrs(to_vmd_dev(dev), sgt, cpu_addr, addr, size, - attrs); -} - -static dma_addr_t vmd_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - return dma_map_page_attrs(to_vmd_dev(dev), page, offset, size, dir, - attrs); -} - -static void vmd_unmap_page(struct device *dev, dma_addr_t addr, size_t size, - enum dma_data_direction dir, unsigned long attrs) -{ - dma_unmap_page_attrs(to_vmd_dev(dev), addr, size, dir, attrs); -} - -static int vmd_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) -{ - return dma_map_sg_attrs(to_vmd_dev(dev), sg, nents, dir, attrs); -} - -static void vmd_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) -{ - dma_unmap_sg_attrs(to_vmd_dev(dev), sg, nents, dir, attrs); -} - -static void vmd_sync_single_for_cpu(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir) -{ - dma_sync_single_for_cpu(to_vmd_dev(dev), addr, size, dir); -} - -static void vmd_sync_single_for_device(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir) -{ - dma_sync_single_for_device(to_vmd_dev(dev), addr, size, dir); -} - -static void vmd_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir) -{ - dma_sync_sg_for_cpu(to_vmd_dev(dev), sg, nents, dir); -} - -static void vmd_sync_sg_for_device(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir) -{ - dma_sync_sg_for_device(to_vmd_dev(dev), sg, nents, dir); -} - -static int vmd_dma_supported(struct device *dev, u64 mask) -{ - return dma_supported(to_vmd_dev(dev), mask); -} - -static u64 vmd_get_required_mask(struct device *dev) -{ - return dma_get_required_mask(to_vmd_dev(dev)); -} - -static void vmd_teardown_dma_ops(struct vmd_dev *vmd) -{ - struct dma_domain *domain = &vmd->dma_domain; - - if (get_dma_ops(&vmd->dev->dev)) - del_dma_domain(domain); -} - -#define ASSIGN_VMD_DMA_OPS(source, dest, fn) \ - do { \ - if (source->fn) \ - dest->fn = vmd_##fn; \ - } while (0) - -static void vmd_setup_dma_ops(struct vmd_dev *vmd) -{ - const struct dma_map_ops *source = get_dma_ops(&vmd->dev->dev); - struct dma_map_ops *dest = &vmd->dma_ops; - struct dma_domain *domain = &vmd->dma_domain; - - domain->domain_nr = vmd->sysdata.domain; - domain->dma_ops = dest; - - if (!source) - return; - ASSIGN_VMD_DMA_OPS(source, dest, alloc); - ASSIGN_VMD_DMA_OPS(source, dest, free); - ASSIGN_VMD_DMA_OPS(source, dest, mmap); - ASSIGN_VMD_DMA_OPS(source, dest, get_sgtable); - ASSIGN_VMD_DMA_OPS(source, dest, map_page); - ASSIGN_VMD_DMA_OPS(source, dest, unmap_page); - ASSIGN_VMD_DMA_OPS(source, dest, map_sg); - ASSIGN_VMD_DMA_OPS(source, dest, unmap_sg); - ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_cpu); - ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_device); - ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_cpu); - ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device); - ASSIGN_VMD_DMA_OPS(source, dest, dma_supported); - ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask); - add_dma_domain(domain); -} -#undef ASSIGN_VMD_DMA_OPS - static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus, unsigned int devfn, int reg, int len) { @@ -679,7 +531,7 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) .parent = res, }; - sd->vmd_domain = true; + sd->vmd_dev = vmd->dev; sd->domain = vmd_find_free_domain(); if (sd->domain < 0) return sd->domain; @@ -709,7 +561,6 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) } vmd_attach_resources(vmd); - vmd_setup_dma_ops(vmd); dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain); pci_scan_child_bus(vmd->bus); @@ -824,7 +675,6 @@ static void vmd_remove(struct pci_dev *dev) pci_stop_root_bus(vmd->bus); pci_remove_root_bus(vmd->bus); vmd_cleanup_srcu(vmd); - vmd_teardown_dma_ops(vmd); vmd_detach_resources(vmd); irq_domain_remove(vmd->irq_domain); } @@ -868,6 +718,10 @@ static const struct pci_device_id vmd_ids[] = { {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0), .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW | VMD_FEAT_HAS_BUS_RESTRICTIONS,}, + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x467f), + .driver_data = VMD_FEAT_HAS_BUS_RESTRICTIONS,}, + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x4c3d), + .driver_data = VMD_FEAT_HAS_BUS_RESTRICTIONS,}, {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B), .driver_data = VMD_FEAT_HAS_BUS_RESTRICTIONS,}, {0,} diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c index d7b2b47bc33e..04565162a449 100644 --- a/drivers/pci/hotplug/pnv_php.c +++ b/drivers/pci/hotplug/pnv_php.c @@ -18,6 +18,9 @@ #define DRIVER_AUTHOR "Gavin Shan, IBM Corporation" #define DRIVER_DESC "PowerPC PowerNV PCI Hotplug Driver" +#define SLOT_WARN(sl, x...) \ + ((sl)->pdev ? pci_warn((sl)->pdev, x) : dev_warn(&(sl)->bus->dev, x)) + struct pnv_php_event { bool added; struct pnv_php_slot *php_slot; @@ -151,17 +154,11 @@ static void pnv_php_rmv_pdns(struct device_node *dn) static void pnv_php_detach_device_nodes(struct device_node *parent) { struct device_node *dn; - int refcount; for_each_child_of_node(parent, dn) { pnv_php_detach_device_nodes(dn); of_node_put(dn); - refcount = kref_read(&dn->kobj.kref); - if (refcount != 1) - pr_warn("Invalid refcount %d on <%pOF>\n", - refcount, dn); - of_detach_node(dn); } } @@ -271,7 +268,7 @@ static int pnv_php_add_devtree(struct pnv_php_slot *php_slot) ret = pnv_pci_get_device_tree(php_slot->dn->phandle, fdt1, 0x10000); if (ret) { - pci_warn(php_slot->pdev, "Error %d getting FDT blob\n", ret); + SLOT_WARN(php_slot, "Error %d getting FDT blob\n", ret); goto free_fdt1; } @@ -285,7 +282,7 @@ static int pnv_php_add_devtree(struct pnv_php_slot *php_slot) dt = of_fdt_unflatten_tree(fdt, php_slot->dn, NULL); if (!dt) { ret = -EINVAL; - pci_warn(php_slot->pdev, "Cannot unflatten FDT\n"); + SLOT_WARN(php_slot, "Cannot unflatten FDT\n"); goto free_fdt; } @@ -295,15 +292,15 @@ static int pnv_php_add_devtree(struct pnv_php_slot *php_slot) ret = pnv_php_populate_changeset(&php_slot->ocs, php_slot->dn); if (ret) { pnv_php_reverse_nodes(php_slot->dn); - pci_warn(php_slot->pdev, "Error %d populating changeset\n", - ret); + SLOT_WARN(php_slot, "Error %d populating changeset\n", + ret); goto free_dt; } php_slot->dn->child = NULL; ret = of_changeset_apply(&php_slot->ocs); if (ret) { - pci_warn(php_slot->pdev, "Error %d applying changeset\n", ret); + SLOT_WARN(php_slot, "Error %d applying changeset\n", ret); goto destroy_changeset; } @@ -342,18 +339,19 @@ int pnv_php_set_slot_power_state(struct hotplug_slot *slot, ret = pnv_pci_set_power_state(php_slot->id, state, &msg); if (ret > 0) { if (be64_to_cpu(msg.params[1]) != php_slot->dn->phandle || - be64_to_cpu(msg.params[2]) != state || - be64_to_cpu(msg.params[3]) != OPAL_SUCCESS) { - pci_warn(php_slot->pdev, "Wrong msg (%lld, %lld, %lld)\n", - be64_to_cpu(msg.params[1]), - be64_to_cpu(msg.params[2]), - be64_to_cpu(msg.params[3])); + be64_to_cpu(msg.params[2]) != state) { + SLOT_WARN(php_slot, "Wrong msg (%lld, %lld, %lld)\n", + be64_to_cpu(msg.params[1]), + be64_to_cpu(msg.params[2]), + be64_to_cpu(msg.params[3])); return -ENOMSG; } + if (be64_to_cpu(msg.params[3]) != OPAL_SUCCESS) { + ret = -ENODEV; + goto error; + } } else if (ret < 0) { - pci_warn(php_slot->pdev, "Error %d powering %s\n", - ret, (state == OPAL_PCI_SLOT_POWER_ON) ? "on" : "off"); - return ret; + goto error; } if (state == OPAL_PCI_SLOT_POWER_OFF || state == OPAL_PCI_SLOT_OFFLINE) @@ -362,6 +360,11 @@ int pnv_php_set_slot_power_state(struct hotplug_slot *slot, ret = pnv_php_add_devtree(php_slot); return ret; + +error: + SLOT_WARN(php_slot, "Error %d powering %s\n", + ret, (state == OPAL_PCI_SLOT_POWER_ON) ? "on" : "off"); + return ret; } EXPORT_SYMBOL_GPL(pnv_php_set_slot_power_state); @@ -378,8 +381,8 @@ static int pnv_php_get_power_state(struct hotplug_slot *slot, u8 *state) */ ret = pnv_pci_get_power_state(php_slot->id, &power_state); if (ret) { - pci_warn(php_slot->pdev, "Error %d getting power status\n", - ret); + SLOT_WARN(php_slot, "Error %d getting power status\n", + ret); } else { *state = power_state; } @@ -402,7 +405,7 @@ static int pnv_php_get_adapter_state(struct hotplug_slot *slot, u8 *state) *state = presence; ret = 0; } else { - pci_warn(php_slot->pdev, "Error %d getting presence\n", ret); + SLOT_WARN(php_slot, "Error %d getting presence\n", ret); } return ret; @@ -566,7 +569,13 @@ static int pnv_php_disable_slot(struct hotplug_slot *slot) struct pnv_php_slot *php_slot = to_pnv_php_slot(slot); int ret; - if (php_slot->state != PNV_PHP_STATE_POPULATED) + /* + * Allow to disable a slot already in the registered state to + * cover cases where the slot couldn't be enabled and never + * reached the populated state + */ + if (php_slot->state != PNV_PHP_STATE_POPULATED && + php_slot->state != PNV_PHP_STATE_REGISTERED) return 0; /* Remove all devices behind the slot */ @@ -675,7 +684,7 @@ static int pnv_php_register_slot(struct pnv_php_slot *php_slot) ret = pci_hp_register(&php_slot->slot, php_slot->bus, php_slot->slot_no, php_slot->name); if (ret) { - pci_warn(php_slot->pdev, "Error %d registering slot\n", ret); + SLOT_WARN(php_slot, "Error %d registering slot\n", ret); return ret; } @@ -728,7 +737,7 @@ static int pnv_php_enable_msix(struct pnv_php_slot *php_slot) /* Enable MSIx */ ret = pci_enable_msix_exact(pdev, &entry, 1); if (ret) { - pci_warn(pdev, "Error %d enabling MSIx\n", ret); + SLOT_WARN(php_slot, "Error %d enabling MSIx\n", ret); return ret; } @@ -778,8 +787,9 @@ static irqreturn_t pnv_php_interrupt(int irq, void *data) (sts & PCI_EXP_SLTSTA_PDC)) { ret = pnv_pci_get_presence_state(php_slot->id, &presence); if (ret) { - pci_warn(pdev, "PCI slot [%s] error %d getting presence (0x%04x), to retry the operation.\n", - php_slot->name, ret, sts); + SLOT_WARN(php_slot, + "PCI slot [%s] error %d getting presence (0x%04x), to retry the operation.\n", + php_slot->name, ret, sts); return IRQ_HANDLED; } @@ -809,8 +819,9 @@ static irqreturn_t pnv_php_interrupt(int irq, void *data) */ event = kzalloc(sizeof(*event), GFP_ATOMIC); if (!event) { - pci_warn(pdev, "PCI slot [%s] missed hotplug event 0x%04x\n", - php_slot->name, sts); + SLOT_WARN(php_slot, + "PCI slot [%s] missed hotplug event 0x%04x\n", + php_slot->name, sts); return IRQ_HANDLED; } @@ -834,7 +845,7 @@ static void pnv_php_init_irq(struct pnv_php_slot *php_slot, int irq) /* Allocate workqueue */ php_slot->wq = alloc_workqueue("pciehp-%s", 0, 0, php_slot->name); if (!php_slot->wq) { - pci_warn(pdev, "Cannot alloc workqueue\n"); + SLOT_WARN(php_slot, "Cannot alloc workqueue\n"); pnv_php_disable_irq(php_slot, true); return; } @@ -858,7 +869,7 @@ static void pnv_php_init_irq(struct pnv_php_slot *php_slot, int irq) php_slot->name, php_slot); if (ret) { pnv_php_disable_irq(php_slot, true); - pci_warn(pdev, "Error %d enabling IRQ %d\n", ret, irq); + SLOT_WARN(php_slot, "Error %d enabling IRQ %d\n", ret, irq); return; } @@ -894,7 +905,7 @@ static void pnv_php_enable_irq(struct pnv_php_slot *php_slot) ret = pci_enable_device(pdev); if (ret) { - pci_warn(pdev, "Error %d enabling device\n", ret); + SLOT_WARN(php_slot, "Error %d enabling device\n", ret); return; } @@ -1009,6 +1020,8 @@ static int __init pnv_php_init(void) for_each_compatible_node(dn, NULL, "ibm,ioda3-phb") pnv_php_register(dn); + for_each_compatible_node(dn, NULL, "ibm,ioda2-npu2-opencapi-phb") + pnv_php_register_one(dn); /* slot directly under the PHB */ return 0; } @@ -1021,6 +1034,9 @@ static void __exit pnv_php_exit(void) for_each_compatible_node(dn, NULL, "ibm,ioda3-phb") pnv_php_unregister(dn); + + for_each_compatible_node(dn, NULL, "ibm,ioda2-npu2-opencapi-phb") + pnv_php_unregister_one(dn); /* slot directly under the PHB */ } module_init(pnv_php_init); diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 1e88fd427757..4d1f392b05f9 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -186,10 +186,10 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id) sprintf(buf, "virtfn%u", id); rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); if (rc) - goto failed2; + goto failed1; rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); if (rc) - goto failed3; + goto failed2; kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); @@ -197,11 +197,10 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id) return 0; -failed3: - sysfs_remove_link(&dev->dev.kobj, buf); failed2: - pci_stop_and_remove_bus_device(virtfn); + sysfs_remove_link(&dev->dev.kobj, buf); failed1: + pci_stop_and_remove_bus_device(virtfn); pci_dev_put(dev); failed0: virtfn_remove_bus(dev->bus, bus); diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 0608aae72ccc..9a8a38384121 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -289,6 +289,9 @@ static const struct pci_p2pdma_whitelist_entry { /* Intel Xeon E7 v3/Xeon E5 v3/Core i7 */ {PCI_VENDOR_ID_INTEL, 0x2f00, REQ_SAME_HOST_BRIDGE}, {PCI_VENDOR_ID_INTEL, 0x2f01, REQ_SAME_HOST_BRIDGE}, + /* Intel SkyLake-E */ + {PCI_VENDOR_ID_INTEL, 0x2030, 0}, + {PCI_VENDOR_ID_INTEL, 0x2020, 0}, {} }; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index df21e3227b57..d828ca835a98 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -131,6 +131,7 @@ bool pci_ats_disabled(void) { return pcie_ats_disabled; } +EXPORT_SYMBOL_GPL(pci_ats_disabled); /* Disable bridge_d3 for all PCIe ports */ static bool pci_bridge_d3_disable; @@ -1372,8 +1373,11 @@ int pci_save_state(struct pci_dev *dev) { int i; /* XXX: 100% dword access ok here? */ - for (i = 0; i < 16; i++) + for (i = 0; i < 16; i++) { pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]); + pci_dbg(dev, "saving config space at offset %#x (reading %#x)\n", + i * 4, dev->saved_config_space[i]); + } dev->state_saved = true; i = pci_save_pcie_state(dev); @@ -5998,7 +6002,8 @@ EXPORT_SYMBOL_GPL(pci_pr3_present); /** * pci_add_dma_alias - Add a DMA devfn alias for a device * @dev: the PCI device for which alias is added - * @devfn: alias slot and function + * @devfn_from: alias slot and function + * @nr_devfns: number of subsequent devfns to alias * * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask * which is used to program permissible bus-devfn source addresses for DMA @@ -6014,18 +6019,29 @@ EXPORT_SYMBOL_GPL(pci_pr3_present); * cannot be left as a userspace activity). DMA aliases should therefore * be configured via quirks, such as the PCI fixup header quirk. */ -void pci_add_dma_alias(struct pci_dev *dev, u8 devfn) +void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, unsigned nr_devfns) { + int devfn_to; + + nr_devfns = min(nr_devfns, (unsigned) MAX_NR_DEVFNS - devfn_from); + devfn_to = devfn_from + nr_devfns - 1; + if (!dev->dma_alias_mask) - dev->dma_alias_mask = bitmap_zalloc(U8_MAX, GFP_KERNEL); + dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL); if (!dev->dma_alias_mask) { pci_warn(dev, "Unable to allocate DMA alias mask\n"); return; } - set_bit(devfn, dev->dma_alias_mask); - pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n", - PCI_SLOT(devfn), PCI_FUNC(devfn)); + bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns); + + if (nr_devfns == 1) + pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n", + PCI_SLOT(devfn_from), PCI_FUNC(devfn_from)); + else if (nr_devfns > 1) + pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n", + PCI_SLOT(devfn_from), PCI_FUNC(devfn_from), + PCI_SLOT(devfn_to), PCI_FUNC(devfn_to)); } bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) @@ -6033,7 +6049,9 @@ bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) return (dev1->dma_alias_mask && test_bit(dev2->devfn, dev1->dma_alias_mask)) || (dev2->dma_alias_mask && - test_bit(dev1->devfn, dev2->dma_alias_mask)); + test_bit(dev1->devfn, dev2->dma_alias_mask)) || + pci_real_dma_dev(dev1) == dev2 || + pci_real_dma_dev(dev2) == dev1; } bool pci_device_is_present(struct pci_dev *pdev) @@ -6057,6 +6075,21 @@ void pci_ignore_hotplug(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pci_ignore_hotplug); +/** + * pci_real_dma_dev - Get PCI DMA device for PCI device + * @dev: the PCI device that may have a PCI DMA alias + * + * Permits the platform to provide architecture-specific functionality to + * devices needing to alias DMA to another PCI device on another PCI bus. If + * the PCI device is on the same bus, it is recommended to use + * pci_add_dma_alias(). This is the default implementation. Architecture + * implementations can override this. + */ +struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev) +{ + return dev; +} + resource_size_t __weak pcibios_default_alignment(void) { return 0; diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index a0a53bd05a0b..6394e7746fb5 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -4,6 +4,9 @@ #include <linux/pci.h> +/* Number of possible devfns: 0.0 to 1f.7 inclusive */ +#define MAX_NR_DEVFNS 256 + #define PCI_FIND_CAP_TTL 48 #define PCI_VSEC_ID_INTEL_TBT 0x1234 /* Thunderbolt */ diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 1ca86f2e0166..4a818b07a1af 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -1445,6 +1445,7 @@ static int aer_probe(struct pcie_device *dev) return -ENOMEM; rpc->rpd = port; + INIT_KFIFO(rpc->aer_fifo); set_service_data(dev, rpc); status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr, diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index b0e6048a9208..01dfc8bb7ca0 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -10,6 +10,8 @@ * Zhang Yanmin (yanmin.zhang@intel.com) */ +#define dev_fmt(fmt) "AER: " fmt + #include <linux/pci.h> #include <linux/module.h> #include <linux/kernel.h> @@ -61,10 +63,12 @@ static int report_error_detected(struct pci_dev *dev, * error callbacks of "any" device in the subtree, and will * exit in the disconnected error state. */ - if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE) + if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE) { vote = PCI_ERS_RESULT_NO_AER_DRIVER; - else + pci_info(dev, "can't recover (no error_detected callback)\n"); + } else { vote = PCI_ERS_RESULT_NONE; + } } else { err_handler = dev->driver->err_handler; vote = err_handler->error_detected(dev, state); @@ -233,12 +237,12 @@ void pcie_do_recovery(struct pci_dev *dev, enum pci_channel_state state, pci_aer_clear_device_status(dev); pci_cleanup_aer_uncorrect_error_status(dev); - pci_info(dev, "AER: Device recovery successful\n"); + pci_info(dev, "device recovery successful\n"); return; failed: pci_uevent_ers(dev, PCI_ERS_RESULT_DISCONNECT); /* TODO: Should kernel panic here? */ - pci_info(dev, "AER: Device recovery failed\n"); + pci_info(dev, "device recovery failed\n"); } diff --git a/drivers/pci/proc.c b/drivers/pci/proc.c index 6ef74bf5013f..bd2b691fa7a3 100644 --- a/drivers/pci/proc.c +++ b/drivers/pci/proc.c @@ -306,19 +306,20 @@ static int proc_bus_pci_release(struct inode *inode, struct file *file) } #endif /* HAVE_PCI_MMAP */ -static const struct file_operations proc_bus_pci_operations = { - .owner = THIS_MODULE, - .llseek = proc_bus_pci_lseek, - .read = proc_bus_pci_read, - .write = proc_bus_pci_write, - .unlocked_ioctl = proc_bus_pci_ioctl, - .compat_ioctl = proc_bus_pci_ioctl, +static const struct proc_ops proc_bus_pci_ops = { + .proc_lseek = proc_bus_pci_lseek, + .proc_read = proc_bus_pci_read, + .proc_write = proc_bus_pci_write, + .proc_ioctl = proc_bus_pci_ioctl, +#ifdef CONFIG_COMPAT + .proc_compat_ioctl = proc_bus_pci_ioctl, +#endif #ifdef HAVE_PCI_MMAP - .open = proc_bus_pci_open, - .release = proc_bus_pci_release, - .mmap = proc_bus_pci_mmap, + .proc_open = proc_bus_pci_open, + .proc_release = proc_bus_pci_release, + .proc_mmap = proc_bus_pci_mmap, #ifdef HAVE_ARCH_PCI_GET_UNMAPPED_AREA - .get_unmapped_area = get_pci_unmapped_area, + .proc_get_unmapped_area = get_pci_unmapped_area, #endif /* HAVE_ARCH_PCI_GET_UNMAPPED_AREA */ #endif /* HAVE_PCI_MMAP */ }; @@ -424,7 +425,7 @@ int pci_proc_attach_device(struct pci_dev *dev) sprintf(name, "%02x.%x", PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn)); e = proc_create_data(name, S_IFREG | S_IRUGO | S_IWUSR, bus->procdir, - &proc_bus_pci_operations, dev); + &proc_bus_pci_ops, dev); if (!e) return -ENOMEM; proc_set_size(e, dev->cfg_size); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index a3a1a0ea64f4..29f473ebf20f 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -1871,19 +1871,40 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2609, quirk_intel_pcie_pm); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x260a, quirk_intel_pcie_pm); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x260b, quirk_intel_pcie_pm); +static void quirk_d3hot_delay(struct pci_dev *dev, unsigned int delay) +{ + if (dev->d3_delay >= delay) + return; + + dev->d3_delay = delay; + pci_info(dev, "extending delay after power-on from D3hot to %d msec\n", + dev->d3_delay); +} + static void quirk_radeon_pm(struct pci_dev *dev) { if (dev->subsystem_vendor == PCI_VENDOR_ID_APPLE && - dev->subsystem_device == 0x00e2) { - if (dev->d3_delay < 20) { - dev->d3_delay = 20; - pci_info(dev, "extending delay after power-on from D3 to %d msec\n", - dev->d3_delay); - } - } + dev->subsystem_device == 0x00e2) + quirk_d3hot_delay(dev, 20); } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x6741, quirk_radeon_pm); +/* + * Ryzen5/7 XHCI controllers fail upon resume from runtime suspend or s2idle. + * https://bugzilla.kernel.org/show_bug.cgi?id=205587 + * + * The kernel attempts to transition these devices to D3cold, but that seems + * to be ineffective on the platforms in question; the PCI device appears to + * remain on in D3hot state. The D3hot-to-D0 transition then requires an + * extended delay in order to succeed. + */ +static void quirk_ryzen_xhci_d3hot(struct pci_dev *dev) +{ + quirk_d3hot_delay(dev, 20); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x15e0, quirk_ryzen_xhci_d3hot); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x15e1, quirk_ryzen_xhci_d3hot); + #ifdef CONFIG_X86_IO_APIC static int dmi_disable_ioapicreroute(const struct dmi_system_id *d) { @@ -2381,32 +2402,6 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5719, quirk_brcm_5719_limit_mrrs); -#ifdef CONFIG_PCIE_IPROC_PLATFORM -static void quirk_paxc_bridge(struct pci_dev *pdev) -{ - /* - * The PCI config space is shared with the PAXC root port and the first - * Ethernet device. So, we need to workaround this by telling the PCI - * code that the bridge is not an Ethernet device. - */ - if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) - pdev->class = PCI_CLASS_BRIDGE_PCI << 8; - - /* - * MPSS is not being set properly (as it is currently 0). This is - * because that area of the PCI config space is hard coded to zero, and - * is not modifiable by firmware. Set this to 2 (e.g., 512 byte MPS) - * so that the MPS can be set to the real max value. - */ - pdev->pcie_mpss = 2; -} -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge); -#endif - /* * Originally in EDAC sources for i82875P: Intel tells BIOS developers to * hide device 6 which configures the overflow device access containing the @@ -3932,7 +3927,7 @@ int pci_dev_specific_reset(struct pci_dev *dev, int probe) static void quirk_dma_func0_alias(struct pci_dev *dev) { if (PCI_FUNC(dev->devfn) != 0) - pci_add_dma_alias(dev, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); + pci_add_dma_alias(dev, PCI_DEVFN(PCI_SLOT(dev->devfn), 0), 1); } /* @@ -3946,7 +3941,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_RICOH, 0xe476, quirk_dma_func0_alias); static void quirk_dma_func1_alias(struct pci_dev *dev) { if (PCI_FUNC(dev->devfn) != 1) - pci_add_dma_alias(dev, PCI_DEVFN(PCI_SLOT(dev->devfn), 1)); + pci_add_dma_alias(dev, PCI_DEVFN(PCI_SLOT(dev->devfn), 1), 1); } /* @@ -4031,7 +4026,7 @@ static void quirk_fixed_dma_alias(struct pci_dev *dev) id = pci_match_id(fixed_dma_alias_tbl, dev); if (id) - pci_add_dma_alias(dev, id->driver_data); + pci_add_dma_alias(dev, id->driver_data, 1); } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ADAPTEC2, 0x0285, quirk_fixed_dma_alias); @@ -4072,9 +4067,9 @@ DECLARE_PCI_FIXUP_HEADER(0x8086, 0x244e, quirk_use_pcie_bridge_dma_alias); */ static void quirk_mic_x200_dma_alias(struct pci_dev *pdev) { - pci_add_dma_alias(pdev, PCI_DEVFN(0x10, 0x0)); - pci_add_dma_alias(pdev, PCI_DEVFN(0x11, 0x0)); - pci_add_dma_alias(pdev, PCI_DEVFN(0x12, 0x3)); + pci_add_dma_alias(pdev, PCI_DEVFN(0x10, 0x0), 1); + pci_add_dma_alias(pdev, PCI_DEVFN(0x11, 0x0), 1); + pci_add_dma_alias(pdev, PCI_DEVFN(0x12, 0x3), 1); } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2260, quirk_mic_x200_dma_alias); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2264, quirk_mic_x200_dma_alias); @@ -4098,13 +4093,8 @@ static void quirk_pex_vca_alias(struct pci_dev *pdev) const unsigned int num_pci_slots = 0x20; unsigned int slot; - for (slot = 0; slot < num_pci_slots; slot++) { - pci_add_dma_alias(pdev, PCI_DEVFN(slot, 0x0)); - pci_add_dma_alias(pdev, PCI_DEVFN(slot, 0x1)); - pci_add_dma_alias(pdev, PCI_DEVFN(slot, 0x2)); - pci_add_dma_alias(pdev, PCI_DEVFN(slot, 0x3)); - pci_add_dma_alias(pdev, PCI_DEVFN(slot, 0x4)); - } + for (slot = 0; slot < num_pci_slots; slot++) + pci_add_dma_alias(pdev, PCI_DEVFN(slot, 0x0), 5); } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2954, quirk_pex_vca_alias); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2955, quirk_pex_vca_alias); @@ -5339,7 +5329,7 @@ static void quirk_switchtec_ntb_dma_alias(struct pci_dev *pdev) pci_dbg(pdev, "Aliasing Partition %d Proxy ID %02x.%d\n", pp, PCI_SLOT(devfn), PCI_FUNC(devfn)); - pci_add_dma_alias(pdev, devfn); + pci_add_dma_alias(pdev, devfn, 1); } } @@ -5380,6 +5370,39 @@ SWITCHTEC_QUIRK(0x8573); /* PFXI 48XG3 */ SWITCHTEC_QUIRK(0x8574); /* PFXI 64XG3 */ SWITCHTEC_QUIRK(0x8575); /* PFXI 80XG3 */ SWITCHTEC_QUIRK(0x8576); /* PFXI 96XG3 */ +SWITCHTEC_QUIRK(0x4000); /* PFX 100XG4 */ +SWITCHTEC_QUIRK(0x4084); /* PFX 84XG4 */ +SWITCHTEC_QUIRK(0x4068); /* PFX 68XG4 */ +SWITCHTEC_QUIRK(0x4052); /* PFX 52XG4 */ +SWITCHTEC_QUIRK(0x4036); /* PFX 36XG4 */ +SWITCHTEC_QUIRK(0x4028); /* PFX 28XG4 */ +SWITCHTEC_QUIRK(0x4100); /* PSX 100XG4 */ +SWITCHTEC_QUIRK(0x4184); /* PSX 84XG4 */ +SWITCHTEC_QUIRK(0x4168); /* PSX 68XG4 */ +SWITCHTEC_QUIRK(0x4152); /* PSX 52XG4 */ +SWITCHTEC_QUIRK(0x4136); /* PSX 36XG4 */ +SWITCHTEC_QUIRK(0x4128); /* PSX 28XG4 */ +SWITCHTEC_QUIRK(0x4200); /* PAX 100XG4 */ +SWITCHTEC_QUIRK(0x4284); /* PAX 84XG4 */ +SWITCHTEC_QUIRK(0x4268); /* PAX 68XG4 */ +SWITCHTEC_QUIRK(0x4252); /* PAX 52XG4 */ +SWITCHTEC_QUIRK(0x4236); /* PAX 36XG4 */ +SWITCHTEC_QUIRK(0x4228); /* PAX 28XG4 */ + +/* + * The PLX NTB uses devfn proxy IDs to move TLPs between NT endpoints. + * These IDs are used to forward responses to the originator on the other + * side of the NTB. Alias all possible IDs to the NTB to permit access when + * the IOMMU is turned on. + */ +static void quirk_plx_ntb_dma_alias(struct pci_dev *pdev) +{ + pci_info(pdev, "Setting PLX NTB proxy ID aliases\n"); + /* PLX NTB may use all 256 devfns */ + pci_add_dma_alias(pdev, 0, 256); +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_PLX, 0x87b0, quirk_plx_ntb_dma_alias); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_PLX, 0x87b1, quirk_plx_ntb_dma_alias); /* * On Lenovo Thinkpad P50 SKUs with a Nvidia Quadro M1000M, the BIOS does diff --git a/drivers/pci/search.c b/drivers/pci/search.c index bade14002fd8..2061672954ee 100644 --- a/drivers/pci/search.c +++ b/drivers/pci/search.c @@ -32,6 +32,12 @@ int pci_for_each_dma_alias(struct pci_dev *pdev, struct pci_bus *bus; int ret; + /* + * The device may have an explicit alias requester ID for DMA where the + * requester is on another PCI bus. + */ + pdev = pci_real_dma_dev(pdev); + ret = fn(pdev, pci_dev_id(pdev), data); if (ret) return ret; @@ -41,9 +47,9 @@ int pci_for_each_dma_alias(struct pci_dev *pdev, * DMA, iterate over that too. */ if (unlikely(pdev->dma_alias_mask)) { - u8 devfn; + unsigned int devfn; - for_each_set_bit(devfn, pdev->dma_alias_mask, U8_MAX) { + for_each_set_bit(devfn, pdev->dma_alias_mask, MAX_NR_DEVFNS) { ret = fn(pdev, PCI_DEVID(pdev->bus->number, devfn), data); if (ret) diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index f279826204eb..f2461bf9243d 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -1803,12 +1803,18 @@ again: /* Restore size and flags */ list_for_each_entry(fail_res, &fail_head, list) { struct resource *res = fail_res->res; + int idx; res->start = fail_res->start; res->end = fail_res->end; res->flags = fail_res->flags; - if (fail_res->dev->subordinate) - res->flags = 0; + + if (pci_is_bridge(fail_res->dev)) { + idx = res - &fail_res->dev->resource[0]; + if (idx >= PCI_BRIDGE_RESOURCES && + idx <= PCI_BRIDGE_RESOURCE_END) + res->flags = 0; + } } free_list(&fail_head); @@ -1832,56 +1838,72 @@ void __init pci_assign_unassigned_resources(void) } } -static void extend_bridge_window(struct pci_dev *bridge, struct resource *res, +static void adjust_bridge_window(struct pci_dev *bridge, struct resource *res, struct list_head *add_list, - resource_size_t available) + resource_size_t new_size) { - struct pci_dev_resource *dev_res; + resource_size_t add_size, size = resource_size(res); if (res->parent) return; - if (resource_size(res) >= available) + if (!new_size) return; - dev_res = res_to_dev_res(add_list, res); - if (!dev_res) - return; - - /* Is there room to extend the window? */ - if (available - resource_size(res) <= dev_res->add_size) - return; + if (new_size > size) { + add_size = new_size - size; + pci_dbg(bridge, "bridge window %pR extended by %pa\n", res, + &add_size); + } else if (new_size < size) { + add_size = size - new_size; + pci_dbg(bridge, "bridge window %pR shrunken by %pa\n", res, + &add_size); + } - dev_res->add_size = available - resource_size(res); - pci_dbg(bridge, "bridge window %pR extended by %pa\n", res, - &dev_res->add_size); + res->end = res->start + new_size - 1; + remove_from_list(add_list, res); } static void pci_bus_distribute_available_resources(struct pci_bus *bus, struct list_head *add_list, - resource_size_t available_io, - resource_size_t available_mmio, - resource_size_t available_mmio_pref) + struct resource io, + struct resource mmio, + struct resource mmio_pref) { - resource_size_t remaining_io, remaining_mmio, remaining_mmio_pref; unsigned int normal_bridges = 0, hotplug_bridges = 0; struct resource *io_res, *mmio_res, *mmio_pref_res; struct pci_dev *dev, *bridge = bus->self; + resource_size_t io_per_hp, mmio_per_hp, mmio_pref_per_hp, align; io_res = &bridge->resource[PCI_BRIDGE_RESOURCES + 0]; mmio_res = &bridge->resource[PCI_BRIDGE_RESOURCES + 1]; mmio_pref_res = &bridge->resource[PCI_BRIDGE_RESOURCES + 2]; /* - * Update additional resource list (add_list) to fill all the - * extra resource space available for this port except the space - * calculated in __pci_bus_size_bridges() which covers all the - * devices currently connected to the port and below. + * The alignment of this bridge is yet to be considered, hence it must + * be done now before extending its bridge window. + */ + align = pci_resource_alignment(bridge, io_res); + if (!io_res->parent && align) + io.start = min(ALIGN(io.start, align), io.end + 1); + + align = pci_resource_alignment(bridge, mmio_res); + if (!mmio_res->parent && align) + mmio.start = min(ALIGN(mmio.start, align), mmio.end + 1); + + align = pci_resource_alignment(bridge, mmio_pref_res); + if (!mmio_pref_res->parent && align) + mmio_pref.start = min(ALIGN(mmio_pref.start, align), + mmio_pref.end + 1); + + /* + * Now that we have adjusted for alignment, update the bridge window + * resources to fill as much remaining resource space as possible. */ - extend_bridge_window(bridge, io_res, add_list, available_io); - extend_bridge_window(bridge, mmio_res, add_list, available_mmio); - extend_bridge_window(bridge, mmio_pref_res, add_list, - available_mmio_pref); + adjust_bridge_window(bridge, io_res, add_list, resource_size(&io)); + adjust_bridge_window(bridge, mmio_res, add_list, resource_size(&mmio)); + adjust_bridge_window(bridge, mmio_pref_res, add_list, + resource_size(&mmio_pref)); /* * Calculate how many hotplug bridges and normal bridges there @@ -1902,11 +1924,9 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus, */ if (hotplug_bridges + normal_bridges == 1) { dev = list_first_entry(&bus->devices, struct pci_dev, bus_list); - if (dev->subordinate) { + if (dev->subordinate) pci_bus_distribute_available_resources(dev->subordinate, - add_list, available_io, available_mmio, - available_mmio_pref); - } + add_list, io, mmio, mmio_pref); return; } @@ -1919,12 +1939,9 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus, * extra space reduced by the minimal required space for the * non-hotplug bridges. */ - remaining_io = available_io; - remaining_mmio = available_mmio; - remaining_mmio_pref = available_mmio_pref; - for_each_pci_bridge(dev, bus) { - const struct resource *res; + resource_size_t used_size; + struct resource *res; if (dev->is_hotplug_bridge) continue; @@ -1934,24 +1951,39 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus, * bridge and devices below it occupy. */ res = &dev->resource[PCI_BRIDGE_RESOURCES + 0]; - if (!res->parent && available_io > resource_size(res)) - remaining_io -= resource_size(res); + align = pci_resource_alignment(dev, res); + align = align ? ALIGN(io.start, align) - io.start : 0; + used_size = align + resource_size(res); + if (!res->parent) + io.start = min(io.start + used_size, io.end + 1); res = &dev->resource[PCI_BRIDGE_RESOURCES + 1]; - if (!res->parent && available_mmio > resource_size(res)) - remaining_mmio -= resource_size(res); + align = pci_resource_alignment(dev, res); + align = align ? ALIGN(mmio.start, align) - mmio.start : 0; + used_size = align + resource_size(res); + if (!res->parent) + mmio.start = min(mmio.start + used_size, mmio.end + 1); res = &dev->resource[PCI_BRIDGE_RESOURCES + 2]; - if (!res->parent && available_mmio_pref > resource_size(res)) - remaining_mmio_pref -= resource_size(res); + align = pci_resource_alignment(dev, res); + align = align ? ALIGN(mmio_pref.start, align) - + mmio_pref.start : 0; + used_size = align + resource_size(res); + if (!res->parent) + mmio_pref.start = min(mmio_pref.start + used_size, + mmio_pref.end + 1); } + io_per_hp = div64_ul(resource_size(&io), hotplug_bridges); + mmio_per_hp = div64_ul(resource_size(&mmio), hotplug_bridges); + mmio_pref_per_hp = div64_ul(resource_size(&mmio_pref), + hotplug_bridges); + /* * Go over devices on this bus and distribute the remaining * resource space between hotplug bridges. */ for_each_pci_bridge(dev, bus) { - resource_size_t align, io, mmio, mmio_pref; struct pci_bus *b; b = dev->subordinate; @@ -1963,42 +1995,31 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus, * hotplug-capable downstream ports taking alignment into * account. */ - align = pci_resource_alignment(bridge, io_res); - io = div64_ul(available_io, hotplug_bridges); - io = min(ALIGN(io, align), remaining_io); - remaining_io -= io; - - align = pci_resource_alignment(bridge, mmio_res); - mmio = div64_ul(available_mmio, hotplug_bridges); - mmio = min(ALIGN(mmio, align), remaining_mmio); - remaining_mmio -= mmio; - - align = pci_resource_alignment(bridge, mmio_pref_res); - mmio_pref = div64_ul(available_mmio_pref, hotplug_bridges); - mmio_pref = min(ALIGN(mmio_pref, align), remaining_mmio_pref); - remaining_mmio_pref -= mmio_pref; + io.end = io.start + io_per_hp - 1; + mmio.end = mmio.start + mmio_per_hp - 1; + mmio_pref.end = mmio_pref.start + mmio_pref_per_hp - 1; pci_bus_distribute_available_resources(b, add_list, io, mmio, mmio_pref); + + io.start += io_per_hp; + mmio.start += mmio_per_hp; + mmio_pref.start += mmio_pref_per_hp; } } static void pci_bridge_distribute_available_resources(struct pci_dev *bridge, struct list_head *add_list) { - resource_size_t available_io, available_mmio, available_mmio_pref; - const struct resource *res; + struct resource available_io, available_mmio, available_mmio_pref; if (!bridge->is_hotplug_bridge) return; /* Take the initial extra resources from the hotplug port */ - res = &bridge->resource[PCI_BRIDGE_RESOURCES + 0]; - available_io = resource_size(res); - res = &bridge->resource[PCI_BRIDGE_RESOURCES + 1]; - available_mmio = resource_size(res); - res = &bridge->resource[PCI_BRIDGE_RESOURCES + 2]; - available_mmio_pref = resource_size(res); + available_io = bridge->resource[PCI_BRIDGE_RESOURCES + 0]; + available_mmio = bridge->resource[PCI_BRIDGE_RESOURCES + 1]; + available_mmio_pref = bridge->resource[PCI_BRIDGE_RESOURCES + 2]; pci_bus_distribute_available_resources(bridge->subordinate, add_list, available_io, @@ -2055,12 +2076,18 @@ again: /* Restore size and flags */ list_for_each_entry(fail_res, &fail_head, list) { struct resource *res = fail_res->res; + int idx; res->start = fail_res->start; res->end = fail_res->end; res->flags = fail_res->flags; - if (fail_res->dev->subordinate) - res->flags = 0; + + if (pci_is_bridge(fail_res->dev)) { + idx = res - &fail_res->dev->resource[0]; + if (idx >= PCI_BRIDGE_RESOURCES && + idx <= PCI_BRIDGE_RESOURCE_END) + res->flags = 0; + } } free_list(&fail_head); diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c index 88091bbfe77f..a823b4b8ef8a 100644 --- a/drivers/pci/switch/switchtec.c +++ b/drivers/pci/switch/switchtec.c @@ -317,8 +317,15 @@ static ssize_t field ## _show(struct device *dev, \ struct device_attribute *attr, char *buf) \ { \ struct switchtec_dev *stdev = to_stdev(dev); \ - return io_string_show(buf, &stdev->mmio_sys_info->field, \ - sizeof(stdev->mmio_sys_info->field)); \ + struct sys_info_regs __iomem *si = stdev->mmio_sys_info; \ + if (stdev->gen == SWITCHTEC_GEN3) \ + return io_string_show(buf, &si->gen3.field, \ + sizeof(si->gen3.field)); \ + else if (stdev->gen == SWITCHTEC_GEN4) \ + return io_string_show(buf, &si->gen4.field, \ + sizeof(si->gen4.field)); \ + else \ + return -ENOTSUPP; \ } \ \ static DEVICE_ATTR_RO(field) @@ -326,13 +333,31 @@ static DEVICE_ATTR_RO(field) DEVICE_ATTR_SYS_INFO_STR(vendor_id); DEVICE_ATTR_SYS_INFO_STR(product_id); DEVICE_ATTR_SYS_INFO_STR(product_revision); -DEVICE_ATTR_SYS_INFO_STR(component_vendor); + +static ssize_t component_vendor_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct switchtec_dev *stdev = to_stdev(dev); + struct sys_info_regs __iomem *si = stdev->mmio_sys_info; + + /* component_vendor field not supported after gen3 */ + if (stdev->gen != SWITCHTEC_GEN3) + return sprintf(buf, "none\n"); + + return io_string_show(buf, &si->gen3.component_vendor, + sizeof(si->gen3.component_vendor)); +} +static DEVICE_ATTR_RO(component_vendor); static ssize_t component_id_show(struct device *dev, struct device_attribute *attr, char *buf) { struct switchtec_dev *stdev = to_stdev(dev); - int id = ioread16(&stdev->mmio_sys_info->component_id); + int id = ioread16(&stdev->mmio_sys_info->gen3.component_id); + + /* component_id field not supported after gen3 */ + if (stdev->gen != SWITCHTEC_GEN3) + return sprintf(buf, "none\n"); return sprintf(buf, "PM%04X\n", id); } @@ -342,7 +367,11 @@ static ssize_t component_revision_show(struct device *dev, struct device_attribute *attr, char *buf) { struct switchtec_dev *stdev = to_stdev(dev); - int rev = ioread8(&stdev->mmio_sys_info->component_revision); + int rev = ioread8(&stdev->mmio_sys_info->gen3.component_revision); + + /* component_revision field not supported after gen3 */ + if (stdev->gen != SWITCHTEC_GEN3) + return sprintf(buf, "255\n"); return sprintf(buf, "%d\n", rev); } @@ -450,6 +479,12 @@ static ssize_t switchtec_dev_write(struct file *filp, const char __user *data, rc = -EFAULT; goto out; } + if (((MRPC_CMD_ID(stuser->cmd) == MRPC_GAS_WRITE) || + (MRPC_CMD_ID(stuser->cmd) == MRPC_GAS_READ)) && + !capable(CAP_SYS_ADMIN)) { + rc = -EPERM; + goto out; + } data += sizeof(stuser->cmd); rc = copy_from_user(&stuser->data, data, size - sizeof(stuser->cmd)); @@ -568,8 +603,15 @@ static int ioctl_flash_info(struct switchtec_dev *stdev, struct switchtec_ioctl_flash_info info = {0}; struct flash_info_regs __iomem *fi = stdev->mmio_flash_info; - info.flash_length = ioread32(&fi->flash_length); - info.num_partitions = SWITCHTEC_IOCTL_NUM_PARTITIONS; + if (stdev->gen == SWITCHTEC_GEN3) { + info.flash_length = ioread32(&fi->gen3.flash_length); + info.num_partitions = SWITCHTEC_NUM_PARTITIONS_GEN3; + } else if (stdev->gen == SWITCHTEC_GEN4) { + info.flash_length = ioread32(&fi->gen4.flash_length); + info.num_partitions = SWITCHTEC_NUM_PARTITIONS_GEN4; + } else { + return -ENOTSUPP; + } if (copy_to_user(uinfo, &info, sizeof(info))) return -EFAULT; @@ -584,75 +626,200 @@ static void set_fw_info_part(struct switchtec_ioctl_flash_part_info *info, info->length = ioread32(&pi->length); } -static int ioctl_flash_part_info(struct switchtec_dev *stdev, - struct switchtec_ioctl_flash_part_info __user *uinfo) +static int flash_part_info_gen3(struct switchtec_dev *stdev, + struct switchtec_ioctl_flash_part_info *info) { - struct switchtec_ioctl_flash_part_info info = {0}; - struct flash_info_regs __iomem *fi = stdev->mmio_flash_info; - struct sys_info_regs __iomem *si = stdev->mmio_sys_info; + struct flash_info_regs_gen3 __iomem *fi = + &stdev->mmio_flash_info->gen3; + struct sys_info_regs_gen3 __iomem *si = &stdev->mmio_sys_info->gen3; u32 active_addr = -1; - if (copy_from_user(&info, uinfo, sizeof(info))) - return -EFAULT; - - switch (info.flash_partition) { + switch (info->flash_partition) { case SWITCHTEC_IOCTL_PART_CFG0: active_addr = ioread32(&fi->active_cfg); - set_fw_info_part(&info, &fi->cfg0); - if (ioread16(&si->cfg_running) == SWITCHTEC_CFG0_RUNNING) - info.active |= SWITCHTEC_IOCTL_PART_RUNNING; + set_fw_info_part(info, &fi->cfg0); + if (ioread16(&si->cfg_running) == SWITCHTEC_GEN3_CFG0_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; break; case SWITCHTEC_IOCTL_PART_CFG1: active_addr = ioread32(&fi->active_cfg); - set_fw_info_part(&info, &fi->cfg1); - if (ioread16(&si->cfg_running) == SWITCHTEC_CFG1_RUNNING) - info.active |= SWITCHTEC_IOCTL_PART_RUNNING; + set_fw_info_part(info, &fi->cfg1); + if (ioread16(&si->cfg_running) == SWITCHTEC_GEN3_CFG1_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; break; case SWITCHTEC_IOCTL_PART_IMG0: active_addr = ioread32(&fi->active_img); - set_fw_info_part(&info, &fi->img0); - if (ioread16(&si->img_running) == SWITCHTEC_IMG0_RUNNING) - info.active |= SWITCHTEC_IOCTL_PART_RUNNING; + set_fw_info_part(info, &fi->img0); + if (ioread16(&si->img_running) == SWITCHTEC_GEN3_IMG0_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; break; case SWITCHTEC_IOCTL_PART_IMG1: active_addr = ioread32(&fi->active_img); - set_fw_info_part(&info, &fi->img1); - if (ioread16(&si->img_running) == SWITCHTEC_IMG1_RUNNING) - info.active |= SWITCHTEC_IOCTL_PART_RUNNING; + set_fw_info_part(info, &fi->img1); + if (ioread16(&si->img_running) == SWITCHTEC_GEN3_IMG1_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_NVLOG: + set_fw_info_part(info, &fi->nvlog); + break; + case SWITCHTEC_IOCTL_PART_VENDOR0: + set_fw_info_part(info, &fi->vendor[0]); + break; + case SWITCHTEC_IOCTL_PART_VENDOR1: + set_fw_info_part(info, &fi->vendor[1]); + break; + case SWITCHTEC_IOCTL_PART_VENDOR2: + set_fw_info_part(info, &fi->vendor[2]); + break; + case SWITCHTEC_IOCTL_PART_VENDOR3: + set_fw_info_part(info, &fi->vendor[3]); + break; + case SWITCHTEC_IOCTL_PART_VENDOR4: + set_fw_info_part(info, &fi->vendor[4]); + break; + case SWITCHTEC_IOCTL_PART_VENDOR5: + set_fw_info_part(info, &fi->vendor[5]); + break; + case SWITCHTEC_IOCTL_PART_VENDOR6: + set_fw_info_part(info, &fi->vendor[6]); + break; + case SWITCHTEC_IOCTL_PART_VENDOR7: + set_fw_info_part(info, &fi->vendor[7]); + break; + default: + return -EINVAL; + } + + if (info->address == active_addr) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + + return 0; +} + +static int flash_part_info_gen4(struct switchtec_dev *stdev, + struct switchtec_ioctl_flash_part_info *info) +{ + struct flash_info_regs_gen4 __iomem *fi = &stdev->mmio_flash_info->gen4; + struct sys_info_regs_gen4 __iomem *si = &stdev->mmio_sys_info->gen4; + struct active_partition_info_gen4 __iomem *af = &fi->active_flag; + + switch (info->flash_partition) { + case SWITCHTEC_IOCTL_PART_MAP_0: + set_fw_info_part(info, &fi->map0); + break; + case SWITCHTEC_IOCTL_PART_MAP_1: + set_fw_info_part(info, &fi->map1); + break; + case SWITCHTEC_IOCTL_PART_KEY_0: + set_fw_info_part(info, &fi->key0); + if (ioread8(&af->key) == SWITCHTEC_GEN4_KEY0_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->key_running) == SWITCHTEC_GEN4_KEY0_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_KEY_1: + set_fw_info_part(info, &fi->key1); + if (ioread8(&af->key) == SWITCHTEC_GEN4_KEY1_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->key_running) == SWITCHTEC_GEN4_KEY1_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_BL2_0: + set_fw_info_part(info, &fi->bl2_0); + if (ioread8(&af->bl2) == SWITCHTEC_GEN4_BL2_0_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->bl2_running) == SWITCHTEC_GEN4_BL2_0_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_BL2_1: + set_fw_info_part(info, &fi->bl2_1); + if (ioread8(&af->bl2) == SWITCHTEC_GEN4_BL2_1_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->bl2_running) == SWITCHTEC_GEN4_BL2_1_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_CFG0: + set_fw_info_part(info, &fi->cfg0); + if (ioread8(&af->cfg) == SWITCHTEC_GEN4_CFG0_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->cfg_running) == SWITCHTEC_GEN4_CFG0_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_CFG1: + set_fw_info_part(info, &fi->cfg1); + if (ioread8(&af->cfg) == SWITCHTEC_GEN4_CFG1_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->cfg_running) == SWITCHTEC_GEN4_CFG1_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_IMG0: + set_fw_info_part(info, &fi->img0); + if (ioread8(&af->img) == SWITCHTEC_GEN4_IMG0_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->img_running) == SWITCHTEC_GEN4_IMG0_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; + break; + case SWITCHTEC_IOCTL_PART_IMG1: + set_fw_info_part(info, &fi->img1); + if (ioread8(&af->img) == SWITCHTEC_GEN4_IMG1_ACTIVE) + info->active |= SWITCHTEC_IOCTL_PART_ACTIVE; + if (ioread16(&si->img_running) == SWITCHTEC_GEN4_IMG1_RUNNING) + info->active |= SWITCHTEC_IOCTL_PART_RUNNING; break; case SWITCHTEC_IOCTL_PART_NVLOG: - set_fw_info_part(&info, &fi->nvlog); + set_fw_info_part(info, &fi->nvlog); break; case SWITCHTEC_IOCTL_PART_VENDOR0: - set_fw_info_part(&info, &fi->vendor[0]); + set_fw_info_part(info, &fi->vendor[0]); break; case SWITCHTEC_IOCTL_PART_VENDOR1: - set_fw_info_part(&info, &fi->vendor[1]); + set_fw_info_part(info, &fi->vendor[1]); break; case SWITCHTEC_IOCTL_PART_VENDOR2: - set_fw_info_part(&info, &fi->vendor[2]); + set_fw_info_part(info, &fi->vendor[2]); break; case SWITCHTEC_IOCTL_PART_VENDOR3: - set_fw_info_part(&info, &fi->vendor[3]); + set_fw_info_part(info, &fi->vendor[3]); break; case SWITCHTEC_IOCTL_PART_VENDOR4: - set_fw_info_part(&info, &fi->vendor[4]); + set_fw_info_part(info, &fi->vendor[4]); break; case SWITCHTEC_IOCTL_PART_VENDOR5: - set_fw_info_part(&info, &fi->vendor[5]); + set_fw_info_part(info, &fi->vendor[5]); break; case SWITCHTEC_IOCTL_PART_VENDOR6: - set_fw_info_part(&info, &fi->vendor[6]); + set_fw_info_part(info, &fi->vendor[6]); break; case SWITCHTEC_IOCTL_PART_VENDOR7: - set_fw_info_part(&info, &fi->vendor[7]); + set_fw_info_part(info, &fi->vendor[7]); break; default: return -EINVAL; } - if (info.address == active_addr) - info.active |= SWITCHTEC_IOCTL_PART_ACTIVE; + return 0; +} + +static int ioctl_flash_part_info(struct switchtec_dev *stdev, + struct switchtec_ioctl_flash_part_info __user *uinfo) +{ + int ret; + struct switchtec_ioctl_flash_part_info info = {0}; + + if (copy_from_user(&info, uinfo, sizeof(info))) + return -EFAULT; + + if (stdev->gen == SWITCHTEC_GEN3) { + ret = flash_part_info_gen3(stdev, &info); + if (ret) + return ret; + } else if (stdev->gen == SWITCHTEC_GEN4) { + ret = flash_part_info_gen4(stdev, &info); + if (ret) + return ret; + } else { + return -ENOTSUPP; + } if (copy_to_user(uinfo, &info, sizeof(info))) return -EFAULT; @@ -683,11 +850,7 @@ static int ioctl_event_summary(struct switchtec_dev *stdev, s->part[i] = reg; } - for (i = 0; i < SWITCHTEC_MAX_PFF_CSR; i++) { - reg = ioread16(&stdev->mmio_pff_csr[i].vendor_id); - if (reg != PCI_VENDOR_ID_MICROSEMI) - break; - + for (i = 0; i < stdev->pff_csr_count; i++) { reg = ioread32(&stdev->mmio_pff_csr[i].pff_event_summary); s->pff[i] = reg; } @@ -751,10 +914,13 @@ static const struct event_reg { EV_PAR(SWITCHTEC_IOCTL_EVENT_MRPC_COMP, mrpc_comp_hdr), EV_PAR(SWITCHTEC_IOCTL_EVENT_MRPC_COMP_ASYNC, mrpc_comp_async_hdr), EV_PAR(SWITCHTEC_IOCTL_EVENT_DYN_PART_BIND_COMP, dyn_binding_hdr), + EV_PAR(SWITCHTEC_IOCTL_EVENT_INTERCOMM_REQ_NOTIFY, + intercomm_notify_hdr), EV_PFF(SWITCHTEC_IOCTL_EVENT_AER_IN_P2P, aer_in_p2p_hdr), EV_PFF(SWITCHTEC_IOCTL_EVENT_AER_IN_VEP, aer_in_vep_hdr), EV_PFF(SWITCHTEC_IOCTL_EVENT_DPC, dpc_hdr), EV_PFF(SWITCHTEC_IOCTL_EVENT_CTS, cts_hdr), + EV_PFF(SWITCHTEC_IOCTL_EVENT_UEC, uec_hdr), EV_PFF(SWITCHTEC_IOCTL_EVENT_HOTPLUG, hotplug_hdr), EV_PFF(SWITCHTEC_IOCTL_EVENT_IER, ier_hdr), EV_PFF(SWITCHTEC_IOCTL_EVENT_THRESH, threshold_hdr), @@ -1181,10 +1347,6 @@ static int mask_event(struct switchtec_dev *stdev, int eid, int idx) if (!(hdr & SWITCHTEC_EVENT_OCCURRED && hdr & SWITCHTEC_EVENT_EN_IRQ)) return 0; - if (eid == SWITCHTEC_IOCTL_EVENT_LINK_STATE || - eid == SWITCHTEC_IOCTL_EVENT_MRPC_COMP) - return 0; - dev_dbg(&stdev->dev, "%s: %d %d %x\n", __func__, eid, idx, hdr); hdr &= ~(SWITCHTEC_EVENT_EN_IRQ | SWITCHTEC_EVENT_OCCURRED); iowrite32(hdr, hdr_reg); @@ -1231,8 +1393,13 @@ static irqreturn_t switchtec_event_isr(int irq, void *dev) check_link_state_events(stdev); - for (eid = 0; eid < SWITCHTEC_IOCTL_MAX_EVENTS; eid++) + for (eid = 0; eid < SWITCHTEC_IOCTL_MAX_EVENTS; eid++) { + if (eid == SWITCHTEC_IOCTL_EVENT_LINK_STATE || + eid == SWITCHTEC_IOCTL_EVENT_MRPC_COMP) + continue; + event_count += mask_all_events(stdev, eid); + } if (event_count) { atomic_inc(&stdev->event_cnt); @@ -1276,7 +1443,7 @@ static int switchtec_init_isr(struct switchtec_dev *stdev) if (nvecs < 0) return nvecs; - event_irq = ioread32(&stdev->mmio_part_cfg->vep_vector_number); + event_irq = ioread16(&stdev->mmio_part_cfg->vep_vector_number); if (event_irq < 0 || event_irq >= nvecs) return -EFAULT; @@ -1324,16 +1491,16 @@ static void init_pff(struct switchtec_dev *stdev) stdev->pff_csr_count = i; reg = ioread32(&pcfg->usp_pff_inst_id); - if (reg < SWITCHTEC_MAX_PFF_CSR) + if (reg < stdev->pff_csr_count) stdev->pff_local[reg] = 1; reg = ioread32(&pcfg->vep_pff_inst_id); - if (reg < SWITCHTEC_MAX_PFF_CSR) + if (reg < stdev->pff_csr_count) stdev->pff_local[reg] = 1; for (i = 0; i < ARRAY_SIZE(pcfg->dsp_pff_inst_id); i++) { reg = ioread32(&pcfg->dsp_pff_inst_id[i]); - if (reg < SWITCHTEC_MAX_PFF_CSR) + if (reg < stdev->pff_csr_count) stdev->pff_local[reg] = 1; } } @@ -1344,12 +1511,13 @@ static int switchtec_init_pci(struct switchtec_dev *stdev, int rc; void __iomem *map; unsigned long res_start, res_len; + u32 __iomem *part_id; rc = pcim_enable_device(pdev); if (rc) return rc; - rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (rc) return rc; @@ -1378,7 +1546,15 @@ static int switchtec_init_pci(struct switchtec_dev *stdev, stdev->mmio_sys_info = stdev->mmio + SWITCHTEC_GAS_SYS_INFO_OFFSET; stdev->mmio_flash_info = stdev->mmio + SWITCHTEC_GAS_FLASH_INFO_OFFSET; stdev->mmio_ntb = stdev->mmio + SWITCHTEC_GAS_NTB_OFFSET; - stdev->partition = ioread8(&stdev->mmio_sys_info->partition_id); + + if (stdev->gen == SWITCHTEC_GEN3) + part_id = &stdev->mmio_sys_info->gen3.partition_id; + else if (stdev->gen == SWITCHTEC_GEN4) + part_id = &stdev->mmio_sys_info->gen4.partition_id; + else + return -ENOTSUPP; + + stdev->partition = ioread8(part_id); stdev->partition_count = ioread8(&stdev->mmio_ntb->partition_count); stdev->mmio_part_cfg_all = stdev->mmio + SWITCHTEC_GAS_PART_CFG_OFFSET; stdev->mmio_part_cfg = &stdev->mmio_part_cfg_all[stdev->partition]; @@ -1420,6 +1596,8 @@ static int switchtec_pci_probe(struct pci_dev *pdev, if (IS_ERR(stdev)) return PTR_ERR(stdev); + stdev->gen = id->driver_data; + rc = switchtec_init_pci(stdev, pdev); if (rc) goto err_put; @@ -1467,7 +1645,7 @@ static void switchtec_pci_remove(struct pci_dev *pdev) put_device(&stdev->dev); } -#define SWITCHTEC_PCI_DEVICE(device_id) \ +#define SWITCHTEC_PCI_DEVICE(device_id, gen) \ { \ .vendor = PCI_VENDOR_ID_MICROSEMI, \ .device = device_id, \ @@ -1475,6 +1653,7 @@ static void switchtec_pci_remove(struct pci_dev *pdev) .subdevice = PCI_ANY_ID, \ .class = (PCI_CLASS_MEMORY_OTHER << 8), \ .class_mask = 0xFFFFFFFF, \ + .driver_data = gen, \ }, \ { \ .vendor = PCI_VENDOR_ID_MICROSEMI, \ @@ -1483,39 +1662,58 @@ static void switchtec_pci_remove(struct pci_dev *pdev) .subdevice = PCI_ANY_ID, \ .class = (PCI_CLASS_BRIDGE_OTHER << 8), \ .class_mask = 0xFFFFFFFF, \ + .driver_data = gen, \ } static const struct pci_device_id switchtec_pci_tbl[] = { - SWITCHTEC_PCI_DEVICE(0x8531), //PFX 24xG3 - SWITCHTEC_PCI_DEVICE(0x8532), //PFX 32xG3 - SWITCHTEC_PCI_DEVICE(0x8533), //PFX 48xG3 - SWITCHTEC_PCI_DEVICE(0x8534), //PFX 64xG3 - SWITCHTEC_PCI_DEVICE(0x8535), //PFX 80xG3 - SWITCHTEC_PCI_DEVICE(0x8536), //PFX 96xG3 - SWITCHTEC_PCI_DEVICE(0x8541), //PSX 24xG3 - SWITCHTEC_PCI_DEVICE(0x8542), //PSX 32xG3 - SWITCHTEC_PCI_DEVICE(0x8543), //PSX 48xG3 - SWITCHTEC_PCI_DEVICE(0x8544), //PSX 64xG3 - SWITCHTEC_PCI_DEVICE(0x8545), //PSX 80xG3 - SWITCHTEC_PCI_DEVICE(0x8546), //PSX 96xG3 - SWITCHTEC_PCI_DEVICE(0x8551), //PAX 24XG3 - SWITCHTEC_PCI_DEVICE(0x8552), //PAX 32XG3 - SWITCHTEC_PCI_DEVICE(0x8553), //PAX 48XG3 - SWITCHTEC_PCI_DEVICE(0x8554), //PAX 64XG3 - SWITCHTEC_PCI_DEVICE(0x8555), //PAX 80XG3 - SWITCHTEC_PCI_DEVICE(0x8556), //PAX 96XG3 - SWITCHTEC_PCI_DEVICE(0x8561), //PFXL 24XG3 - SWITCHTEC_PCI_DEVICE(0x8562), //PFXL 32XG3 - SWITCHTEC_PCI_DEVICE(0x8563), //PFXL 48XG3 - SWITCHTEC_PCI_DEVICE(0x8564), //PFXL 64XG3 - SWITCHTEC_PCI_DEVICE(0x8565), //PFXL 80XG3 - SWITCHTEC_PCI_DEVICE(0x8566), //PFXL 96XG3 - SWITCHTEC_PCI_DEVICE(0x8571), //PFXI 24XG3 - SWITCHTEC_PCI_DEVICE(0x8572), //PFXI 32XG3 - SWITCHTEC_PCI_DEVICE(0x8573), //PFXI 48XG3 - SWITCHTEC_PCI_DEVICE(0x8574), //PFXI 64XG3 - SWITCHTEC_PCI_DEVICE(0x8575), //PFXI 80XG3 - SWITCHTEC_PCI_DEVICE(0x8576), //PFXI 96XG3 + SWITCHTEC_PCI_DEVICE(0x8531, SWITCHTEC_GEN3), //PFX 24xG3 + SWITCHTEC_PCI_DEVICE(0x8532, SWITCHTEC_GEN3), //PFX 32xG3 + SWITCHTEC_PCI_DEVICE(0x8533, SWITCHTEC_GEN3), //PFX 48xG3 + SWITCHTEC_PCI_DEVICE(0x8534, SWITCHTEC_GEN3), //PFX 64xG3 + SWITCHTEC_PCI_DEVICE(0x8535, SWITCHTEC_GEN3), //PFX 80xG3 + SWITCHTEC_PCI_DEVICE(0x8536, SWITCHTEC_GEN3), //PFX 96xG3 + SWITCHTEC_PCI_DEVICE(0x8541, SWITCHTEC_GEN3), //PSX 24xG3 + SWITCHTEC_PCI_DEVICE(0x8542, SWITCHTEC_GEN3), //PSX 32xG3 + SWITCHTEC_PCI_DEVICE(0x8543, SWITCHTEC_GEN3), //PSX 48xG3 + SWITCHTEC_PCI_DEVICE(0x8544, SWITCHTEC_GEN3), //PSX 64xG3 + SWITCHTEC_PCI_DEVICE(0x8545, SWITCHTEC_GEN3), //PSX 80xG3 + SWITCHTEC_PCI_DEVICE(0x8546, SWITCHTEC_GEN3), //PSX 96xG3 + SWITCHTEC_PCI_DEVICE(0x8551, SWITCHTEC_GEN3), //PAX 24XG3 + SWITCHTEC_PCI_DEVICE(0x8552, SWITCHTEC_GEN3), //PAX 32XG3 + SWITCHTEC_PCI_DEVICE(0x8553, SWITCHTEC_GEN3), //PAX 48XG3 + SWITCHTEC_PCI_DEVICE(0x8554, SWITCHTEC_GEN3), //PAX 64XG3 + SWITCHTEC_PCI_DEVICE(0x8555, SWITCHTEC_GEN3), //PAX 80XG3 + SWITCHTEC_PCI_DEVICE(0x8556, SWITCHTEC_GEN3), //PAX 96XG3 + SWITCHTEC_PCI_DEVICE(0x8561, SWITCHTEC_GEN3), //PFXL 24XG3 + SWITCHTEC_PCI_DEVICE(0x8562, SWITCHTEC_GEN3), //PFXL 32XG3 + SWITCHTEC_PCI_DEVICE(0x8563, SWITCHTEC_GEN3), //PFXL 48XG3 + SWITCHTEC_PCI_DEVICE(0x8564, SWITCHTEC_GEN3), //PFXL 64XG3 + SWITCHTEC_PCI_DEVICE(0x8565, SWITCHTEC_GEN3), //PFXL 80XG3 + SWITCHTEC_PCI_DEVICE(0x8566, SWITCHTEC_GEN3), //PFXL 96XG3 + SWITCHTEC_PCI_DEVICE(0x8571, SWITCHTEC_GEN3), //PFXI 24XG3 + SWITCHTEC_PCI_DEVICE(0x8572, SWITCHTEC_GEN3), //PFXI 32XG3 + SWITCHTEC_PCI_DEVICE(0x8573, SWITCHTEC_GEN3), //PFXI 48XG3 + SWITCHTEC_PCI_DEVICE(0x8574, SWITCHTEC_GEN3), //PFXI 64XG3 + SWITCHTEC_PCI_DEVICE(0x8575, SWITCHTEC_GEN3), //PFXI 80XG3 + SWITCHTEC_PCI_DEVICE(0x8576, SWITCHTEC_GEN3), //PFXI 96XG3 + SWITCHTEC_PCI_DEVICE(0x4000, SWITCHTEC_GEN4), //PFX 100XG4 + SWITCHTEC_PCI_DEVICE(0x4084, SWITCHTEC_GEN4), //PFX 84XG4 + SWITCHTEC_PCI_DEVICE(0x4068, SWITCHTEC_GEN4), //PFX 68XG4 + SWITCHTEC_PCI_DEVICE(0x4052, SWITCHTEC_GEN4), //PFX 52XG4 + SWITCHTEC_PCI_DEVICE(0x4036, SWITCHTEC_GEN4), //PFX 36XG4 + SWITCHTEC_PCI_DEVICE(0x4028, SWITCHTEC_GEN4), //PFX 28XG4 + SWITCHTEC_PCI_DEVICE(0x4100, SWITCHTEC_GEN4), //PSX 100XG4 + SWITCHTEC_PCI_DEVICE(0x4184, SWITCHTEC_GEN4), //PSX 84XG4 + SWITCHTEC_PCI_DEVICE(0x4168, SWITCHTEC_GEN4), //PSX 68XG4 + SWITCHTEC_PCI_DEVICE(0x4152, SWITCHTEC_GEN4), //PSX 52XG4 + SWITCHTEC_PCI_DEVICE(0x4136, SWITCHTEC_GEN4), //PSX 36XG4 + SWITCHTEC_PCI_DEVICE(0x4128, SWITCHTEC_GEN4), //PSX 28XG4 + SWITCHTEC_PCI_DEVICE(0x4200, SWITCHTEC_GEN4), //PAX 100XG4 + SWITCHTEC_PCI_DEVICE(0x4284, SWITCHTEC_GEN4), //PAX 84XG4 + SWITCHTEC_PCI_DEVICE(0x4268, SWITCHTEC_GEN4), //PAX 68XG4 + SWITCHTEC_PCI_DEVICE(0x4252, SWITCHTEC_GEN4), //PAX 52XG4 + SWITCHTEC_PCI_DEVICE(0x4236, SWITCHTEC_GEN4), //PAX 36XG4 + SWITCHTEC_PCI_DEVICE(0x4228, SWITCHTEC_GEN4), //PAX 28XG4 {0} }; MODULE_DEVICE_TABLE(pci, switchtec_pci_tbl); diff --git a/drivers/pcmcia/i82092.c b/drivers/pcmcia/i82092.c index aad8a46605be..85887d885b5f 100644 --- a/drivers/pcmcia/i82092.c +++ b/drivers/pcmcia/i82092.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* +/* * Driver for Intel I82092AA PCI-PCMCIA bridge. * * (C) 2001 Red Hat, Inc. @@ -18,7 +18,7 @@ #include <pcmcia/ss.h> -#include <asm/io.h> +#include <linux/io.h> #include "i82092aa.h" #include "i82365.h" @@ -33,16 +33,16 @@ static const struct pci_device_id i82092aa_pci_ids[] = { MODULE_DEVICE_TABLE(pci, i82092aa_pci_ids); static struct pci_driver i82092aa_pci_driver = { - .name = "i82092aa", - .id_table = i82092aa_pci_ids, - .probe = i82092aa_pci_probe, - .remove = i82092aa_pci_remove, + .name = "i82092aa", + .id_table = i82092aa_pci_ids, + .probe = i82092aa_pci_probe, + .remove = i82092aa_pci_remove, }; /* the pccard structure and its functions */ static struct pccard_operations i82092aa_operations = { - .init = i82092aa_init, + .init = i82092aa_init, .get_status = i82092aa_get_status, .set_socket = i82092aa_set_socket, .set_io_map = i82092aa_set_io_map, @@ -53,57 +53,63 @@ static struct pccard_operations i82092aa_operations = { struct socket_info { int number; - int card_state; /* 0 = no socket, - 1 = empty socket, - 2 = card but not initialized, - 3 = operational card */ - unsigned int io_base; /* base io address of the socket */ - + int card_state; + /* 0 = no socket, + * 1 = empty socket, + * 2 = card but not initialized, + * 3 = operational card + */ + unsigned int io_base; /* base io address of the socket */ + struct pcmcia_socket socket; struct pci_dev *dev; /* The PCI device for the socket */ }; #define MAX_SOCKETS 4 static struct socket_info sockets[MAX_SOCKETS]; -static int socket_count; /* shortcut */ +static int socket_count; /* shortcut */ -static int i82092aa_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) +static int i82092aa_pci_probe(struct pci_dev *dev, + const struct pci_device_id *id) { unsigned char configbyte; int i, ret; - - enter("i82092aa_pci_probe"); - - if ((ret = pci_enable_device(dev))) + + ret = pci_enable_device(dev); + if (ret) return ret; - - pci_read_config_byte(dev, 0x40, &configbyte); /* PCI Configuration Control */ - switch(configbyte&6) { - case 0: - socket_count = 2; - break; - case 2: - socket_count = 1; - break; - case 4: - case 6: - socket_count = 4; - break; - - default: - printk(KERN_ERR "i82092aa: Oops, you did something we didn't think of.\n"); - ret = -EIO; - goto err_out_disable; + + /* PCI Configuration Control */ + pci_read_config_byte(dev, 0x40, &configbyte); + + switch (configbyte&6) { + case 0: + socket_count = 2; + break; + case 2: + socket_count = 1; + break; + case 4: + case 6: + socket_count = 4; + break; + + default: + dev_err(&dev->dev, + "Oops, you did something we didn't think of.\n"); + ret = -EIO; + goto err_out_disable; } - printk(KERN_INFO "i82092aa: configured as a %d socket device.\n", socket_count); + dev_info(&dev->dev, "configured as a %d socket device.\n", + socket_count); if (!request_region(pci_resource_start(dev, 0), 2, "i82092aa")) { ret = -EBUSY; goto err_out_disable; } - - for (i = 0;i<socket_count;i++) { + + for (i = 0; i < socket_count; i++) { sockets[i].card_state = 1; /* 1 = present but empty */ sockets[i].io_base = pci_resource_start(dev, 0); sockets[i].socket.features |= SS_CAP_PCCARD; @@ -114,65 +120,65 @@ static int i82092aa_pci_probe(struct pci_dev *dev, const struct pci_device_id *i sockets[i].socket.owner = THIS_MODULE; sockets[i].number = i; - + if (card_present(i)) { sockets[i].card_state = 3; - dev_dbg(&dev->dev, "i82092aa: slot %i is occupied\n", i); + dev_dbg(&dev->dev, "slot %i is occupied\n", i); } else { - dev_dbg(&dev->dev, "i82092aa: slot %i is vacant\n", i); + dev_dbg(&dev->dev, "slot %i is vacant\n", i); } } - - /* Now, specifiy that all interrupts are to be done as PCI interrupts */ - configbyte = 0xFF; /* bitmask, one bit per event, 1 = PCI interrupt, 0 = ISA interrupt */ - pci_write_config_byte(dev, 0x50, configbyte); /* PCI Interrupt Routing Register */ + + /* Now, specifiy that all interrupts are to be done as PCI interrupts + * bitmask, one bit per event, 1 = PCI interrupt, 0 = ISA interrupt + */ + configbyte = 0xFF; + + /* PCI Interrupt Routing Register */ + pci_write_config_byte(dev, 0x50, configbyte); /* Register the interrupt handler */ dev_dbg(&dev->dev, "Requesting interrupt %i\n", dev->irq); - if ((ret = request_irq(dev->irq, i82092aa_interrupt, IRQF_SHARED, "i82092aa", i82092aa_interrupt))) { - printk(KERN_ERR "i82092aa: Failed to register IRQ %d, aborting\n", dev->irq); + ret = request_irq(dev->irq, i82092aa_interrupt, IRQF_SHARED, + "i82092aa", i82092aa_interrupt); + if (ret) { + dev_err(&dev->dev, "Failed to register IRQ %d, aborting\n", + dev->irq); goto err_out_free_res; } - for (i = 0; i<socket_count; i++) { + for (i = 0; i < socket_count; i++) { sockets[i].socket.dev.parent = &dev->dev; sockets[i].socket.ops = &i82092aa_operations; sockets[i].socket.resource_ops = &pccard_nonstatic_ops; ret = pcmcia_register_socket(&sockets[i].socket); - if (ret) { + if (ret) goto err_out_free_sockets; - } } - leave("i82092aa_pci_probe"); return 0; err_out_free_sockets: if (i) { - for (i--;i>=0;i--) { + for (i--; i >= 0; i--) pcmcia_unregister_socket(&sockets[i].socket); - } } free_irq(dev->irq, i82092aa_interrupt); err_out_free_res: release_region(pci_resource_start(dev, 0), 2); err_out_disable: pci_disable_device(dev); - return ret; + return ret; } static void i82092aa_pci_remove(struct pci_dev *dev) { int i; - enter("i82092aa_pci_remove"); - free_irq(dev->irq, i82092aa_interrupt); for (i = 0; i < socket_count; i++) pcmcia_unregister_socket(&sockets[i].socket); - - leave("i82092aa_pci_remove"); } static DEFINE_SPINLOCK(port_lock); @@ -184,44 +190,27 @@ static unsigned char indirect_read(int socket, unsigned short reg) unsigned short int port; unsigned char val; unsigned long flags; - spin_lock_irqsave(&port_lock,flags); + + spin_lock_irqsave(&port_lock, flags); reg += socket * 0x40; port = sockets[socket].io_base; - outb(reg,port); + outb(reg, port); val = inb(port+1); - spin_unlock_irqrestore(&port_lock,flags); + spin_unlock_irqrestore(&port_lock, flags); return val; } -#if 0 -static unsigned short indirect_read16(int socket, unsigned short reg) -{ - unsigned short int port; - unsigned short tmp; - unsigned long flags; - spin_lock_irqsave(&port_lock,flags); - reg = reg + socket * 0x40; - port = sockets[socket].io_base; - outb(reg,port); - tmp = inb(port+1); - reg++; - outb(reg,port); - tmp = tmp | (inb(port+1)<<8); - spin_unlock_irqrestore(&port_lock,flags); - return tmp; -} -#endif - static void indirect_write(int socket, unsigned short reg, unsigned char value) { unsigned short int port; unsigned long flags; - spin_lock_irqsave(&port_lock,flags); + + spin_lock_irqsave(&port_lock, flags); reg = reg + socket * 0x40; - port = sockets[socket].io_base; - outb(reg,port); - outb(value,port+1); - spin_unlock_irqrestore(&port_lock,flags); + port = sockets[socket].io_base; + outb(reg, port); + outb(value, port+1); + spin_unlock_irqrestore(&port_lock, flags); } static void indirect_setbit(int socket, unsigned short reg, unsigned char mask) @@ -229,53 +218,58 @@ static void indirect_setbit(int socket, unsigned short reg, unsigned char mask) unsigned short int port; unsigned char val; unsigned long flags; - spin_lock_irqsave(&port_lock,flags); + + spin_lock_irqsave(&port_lock, flags); reg = reg + socket * 0x40; - port = sockets[socket].io_base; - outb(reg,port); + port = sockets[socket].io_base; + outb(reg, port); val = inb(port+1); val |= mask; - outb(reg,port); - outb(val,port+1); - spin_unlock_irqrestore(&port_lock,flags); + outb(reg, port); + outb(val, port+1); + spin_unlock_irqrestore(&port_lock, flags); } -static void indirect_resetbit(int socket, unsigned short reg, unsigned char mask) +static void indirect_resetbit(int socket, + unsigned short reg, unsigned char mask) { unsigned short int port; unsigned char val; unsigned long flags; - spin_lock_irqsave(&port_lock,flags); + + spin_lock_irqsave(&port_lock, flags); reg = reg + socket * 0x40; - port = sockets[socket].io_base; - outb(reg,port); + port = sockets[socket].io_base; + outb(reg, port); val = inb(port+1); val &= ~mask; - outb(reg,port); - outb(val,port+1); - spin_unlock_irqrestore(&port_lock,flags); + outb(reg, port); + outb(val, port+1); + spin_unlock_irqrestore(&port_lock, flags); } -static void indirect_write16(int socket, unsigned short reg, unsigned short value) +static void indirect_write16(int socket, + unsigned short reg, unsigned short value) { unsigned short int port; unsigned char val; unsigned long flags; - spin_lock_irqsave(&port_lock,flags); + + spin_lock_irqsave(&port_lock, flags); reg = reg + socket * 0x40; - port = sockets[socket].io_base; - - outb(reg,port); + port = sockets[socket].io_base; + + outb(reg, port); val = value & 255; - outb(val,port+1); - + outb(val, port+1); + reg++; - - outb(reg,port); + + outb(reg, port); val = value>>8; - outb(val,port+1); - spin_unlock_irqrestore(&port_lock,flags); + outb(val, port+1); + spin_unlock_irqrestore(&port_lock, flags); } /* simple helper functions */ @@ -284,12 +278,12 @@ static int cycle_time = 120; static int to_cycles(int ns) { - if (cycle_time!=0) + if (cycle_time != 0) return ns/cycle_time; else return 0; } - + /* Interrupt handler functionality */ @@ -299,58 +293,61 @@ static irqreturn_t i82092aa_interrupt(int irq, void *dev) int loopcount = 0; int handled = 0; - unsigned int events, active=0; - -/* enter("i82092aa_interrupt");*/ - + unsigned int events, active = 0; + while (1) { loopcount++; - if (loopcount>20) { - printk(KERN_ERR "i82092aa: infinite eventloop in interrupt \n"); + if (loopcount > 20) { + pr_err("i82092aa: infinite eventloop in interrupt\n"); break; } - + active = 0; - - for (i=0;i<socket_count;i++) { + + for (i = 0; i < socket_count; i++) { int csc; - if (sockets[i].card_state==0) /* Inactive socket, should not happen */ + + /* Inactive socket, should not happen */ + if (sockets[i].card_state == 0) + continue; + + /* card status change register */ + csc = indirect_read(i, I365_CSC); + + if (csc == 0) /* no events on this socket */ continue; - - csc = indirect_read(i,I365_CSC); /* card status change register */ - - if (csc==0) /* no events on this socket */ - continue; handled = 1; events = 0; - + if (csc & I365_CSC_DETECT) { events |= SS_DETECT; - printk("Card detected in socket %i!\n",i); - } - - if (indirect_read(i,I365_INTCTL) & I365_PC_IOCARD) { + dev_info(&sockets[i].dev->dev, + "Card detected in socket %i!\n", i); + } + + if (indirect_read(i, I365_INTCTL) & I365_PC_IOCARD) { /* For IO/CARDS, bit 0 means "read the card" */ - events |= (csc & I365_CSC_STSCHG) ? SS_STSCHG : 0; + if (csc & I365_CSC_STSCHG) + events |= SS_STSCHG; } else { /* Check for battery/ready events */ - events |= (csc & I365_CSC_BVD1) ? SS_BATDEAD : 0; - events |= (csc & I365_CSC_BVD2) ? SS_BATWARN : 0; - events |= (csc & I365_CSC_READY) ? SS_READY : 0; + if (csc & I365_CSC_BVD1) + events |= SS_BATDEAD; + if (csc & I365_CSC_BVD2) + events |= SS_BATWARN; + if (csc & I365_CSC_READY) + events |= SS_READY; } - - if (events) { + + if (events) pcmcia_parse_events(&sockets[i].socket, events); - } active |= events; } - - if (active==0) /* no more events to handle */ - break; - + + if (active == 0) /* no more events to handle */ + break; } return IRQ_RETVAL(handled); -/* leave("i82092aa_interrupt");*/ } @@ -358,80 +355,67 @@ static irqreturn_t i82092aa_interrupt(int irq, void *dev) /* socket functions */ static int card_present(int socketno) -{ +{ unsigned int val; - enter("card_present"); - - if ((socketno<0) || (socketno >= MAX_SOCKETS)) + + if ((socketno < 0) || (socketno >= MAX_SOCKETS)) return 0; if (sockets[socketno].io_base == 0) return 0; - + val = indirect_read(socketno, 1); /* Interface status register */ - if ((val&12)==12) { - leave("card_present 1"); + if ((val&12) == 12) return 1; - } - - leave("card_present 0"); + return 0; } static void set_bridge_state(int sock) { - enter("set_bridge_state"); - indirect_write(sock, I365_GBLCTL,0x00); - indirect_write(sock, I365_GENCTL,0x00); - - indirect_setbit(sock, I365_INTCTL,0x08); - leave("set_bridge_state"); -} - - + indirect_write(sock, I365_GBLCTL, 0x00); + indirect_write(sock, I365_GENCTL, 0x00); + indirect_setbit(sock, I365_INTCTL, 0x08); +} - static int i82092aa_init(struct pcmcia_socket *sock) { int i; struct resource res = { .start = 0, .end = 0x0fff }; - pccard_io_map io = { 0, 0, 0, 0, 1 }; + pccard_io_map io = { 0, 0, 0, 0, 1 }; pccard_mem_map mem = { .res = &res, }; - - enter("i82092aa_init"); - - for (i = 0; i < 2; i++) { - io.map = i; - i82092aa_set_io_map(sock, &io); + + for (i = 0; i < 2; i++) { + io.map = i; + i82092aa_set_io_map(sock, &io); } - for (i = 0; i < 5; i++) { - mem.map = i; - i82092aa_set_mem_map(sock, &mem); + for (i = 0; i < 5; i++) { + mem.map = i; + i82092aa_set_mem_map(sock, &mem); } - - leave("i82092aa_init"); + return 0; } - + static int i82092aa_get_status(struct pcmcia_socket *socket, u_int *value) { - unsigned int sock = container_of(socket, struct socket_info, socket)->number; + unsigned int sock = container_of(socket, + struct socket_info, socket)->number; unsigned int status; - - enter("i82092aa_get_status"); - - status = indirect_read(sock,I365_STATUS); /* Interface Status Register */ + + /* Interface Status Register */ + status = indirect_read(sock, I365_STATUS); + *value = 0; - - if ((status & I365_CS_DETECT) == I365_CS_DETECT) { + + if ((status & I365_CS_DETECT) == I365_CS_DETECT) *value |= SS_DETECT; - } - + /* IO cards have a different meaning of bits 0,1 */ /* Also notice the inverse-logic on the bits */ - if (indirect_read(sock, I365_INTCTL) & I365_PC_IOCARD) { + if (indirect_read(sock, I365_INTCTL) & I365_PC_IOCARD) { /* IO card */ if (!(status & I365_CS_STSCHG)) *value |= SS_STSCHG; @@ -441,246 +425,238 @@ static int i82092aa_get_status(struct pcmcia_socket *socket, u_int *value) if (!(status & I365_CS_BVD2)) *value |= SS_BATWARN; } - + if (status & I365_CS_WRPROT) (*value) |= SS_WRPROT; /* card is write protected */ - + if (status & I365_CS_READY) (*value) |= SS_READY; /* card is not busy */ - + if (status & I365_CS_POWERON) (*value) |= SS_POWERON; /* power is applied to the card */ - leave("i82092aa_get_status"); return 0; } -static int i82092aa_set_socket(struct pcmcia_socket *socket, socket_state_t *state) +static int i82092aa_set_socket(struct pcmcia_socket *socket, + socket_state_t *state) { - unsigned int sock = container_of(socket, struct socket_info, socket)->number; + struct socket_info *sock_info = container_of(socket, struct socket_info, + socket); + unsigned int sock = sock_info->number; unsigned char reg; - - enter("i82092aa_set_socket"); - + /* First, set the global controller options */ - + set_bridge_state(sock); - + /* Values for the IGENC register */ - + reg = 0; - if (!(state->flags & SS_RESET)) /* The reset bit has "inverse" logic */ - reg = reg | I365_PC_RESET; - if (state->flags & SS_IOCARD) + + /* The reset bit has "inverse" logic */ + if (!(state->flags & SS_RESET)) + reg = reg | I365_PC_RESET; + if (state->flags & SS_IOCARD) reg = reg | I365_PC_IOCARD; - - indirect_write(sock,I365_INTCTL,reg); /* IGENC, Interrupt and General Control Register */ - + + /* IGENC, Interrupt and General Control Register */ + indirect_write(sock, I365_INTCTL, reg); + /* Power registers */ - + reg = I365_PWR_NORESET; /* default: disable resetdrv on resume */ - + if (state->flags & SS_PWR_AUTO) { - printk("Auto power\n"); + dev_info(&sock_info->dev->dev, "Auto power\n"); reg |= I365_PWR_AUTO; /* automatic power mngmnt */ } if (state->flags & SS_OUTPUT_ENA) { - printk("Power Enabled \n"); + dev_info(&sock_info->dev->dev, "Power Enabled\n"); reg |= I365_PWR_OUT; /* enable power */ } - + switch (state->Vcc) { - case 0: - break; - case 50: - printk("setting voltage to Vcc to 5V on socket %i\n",sock); - reg |= I365_VCC_5V; - break; - default: - printk("i82092aa: i82092aa_set_socket called with invalid VCC power value: %i ", state->Vcc); - leave("i82092aa_set_socket"); - return -EINVAL; + case 0: + break; + case 50: + dev_info(&sock_info->dev->dev, + "setting voltage to Vcc to 5V on socket %i\n", + sock); + reg |= I365_VCC_5V; + break; + default: + dev_err(&sock_info->dev->dev, + "%s called with invalid VCC power value: %i", + __func__, state->Vcc); + return -EINVAL; } - - + switch (state->Vpp) { - case 0: - printk("not setting Vpp on socket %i\n",sock); - break; - case 50: - printk("setting Vpp to 5.0 for socket %i\n",sock); - reg |= I365_VPP1_5V | I365_VPP2_5V; - break; - case 120: - printk("setting Vpp to 12.0\n"); - reg |= I365_VPP1_12V | I365_VPP2_12V; - break; - default: - printk("i82092aa: i82092aa_set_socket called with invalid VPP power value: %i ", state->Vcc); - leave("i82092aa_set_socket"); - return -EINVAL; + case 0: + dev_info(&sock_info->dev->dev, + "not setting Vpp on socket %i\n", sock); + break; + case 50: + dev_info(&sock_info->dev->dev, + "setting Vpp to 5.0 for socket %i\n", sock); + reg |= I365_VPP1_5V | I365_VPP2_5V; + break; + case 120: + dev_info(&sock_info->dev->dev, "setting Vpp to 12.0\n"); + reg |= I365_VPP1_12V | I365_VPP2_12V; + break; + default: + dev_err(&sock_info->dev->dev, + "%s called with invalid VPP power value: %i", + __func__, state->Vcc); + return -EINVAL; } - - if (reg != indirect_read(sock,I365_POWER)) /* only write if changed */ - indirect_write(sock,I365_POWER,reg); - + + if (reg != indirect_read(sock, I365_POWER)) /* only write if changed */ + indirect_write(sock, I365_POWER, reg); + /* Enable specific interrupt events */ - + reg = 0x00; - if (state->csc_mask & SS_DETECT) { + if (state->csc_mask & SS_DETECT) reg |= I365_CSC_DETECT; - } if (state->flags & SS_IOCARD) { if (state->csc_mask & SS_STSCHG) reg |= I365_CSC_STSCHG; } else { - if (state->csc_mask & SS_BATDEAD) + if (state->csc_mask & SS_BATDEAD) reg |= I365_CSC_BVD1; - if (state->csc_mask & SS_BATWARN) + if (state->csc_mask & SS_BATWARN) reg |= I365_CSC_BVD2; - if (state->csc_mask & SS_READY) - reg |= I365_CSC_READY; - + if (state->csc_mask & SS_READY) + reg |= I365_CSC_READY; + } - - /* now write the value and clear the (probably bogus) pending stuff by doing a dummy read*/ - - indirect_write(sock,I365_CSCINT,reg); - (void)indirect_read(sock,I365_CSC); - leave("i82092aa_set_socket"); + /* now write the value and clear the (probably bogus) pending stuff + * by doing a dummy read + */ + + indirect_write(sock, I365_CSCINT, reg); + (void)indirect_read(sock, I365_CSC); + return 0; } -static int i82092aa_set_io_map(struct pcmcia_socket *socket, struct pccard_io_map *io) +static int i82092aa_set_io_map(struct pcmcia_socket *socket, + struct pccard_io_map *io) { - unsigned int sock = container_of(socket, struct socket_info, socket)->number; + struct socket_info *sock_info = container_of(socket, struct socket_info, + socket); + unsigned int sock = sock_info->number; unsigned char map, ioctl; - - enter("i82092aa_set_io_map"); - + map = io->map; - - /* Check error conditions */ - if (map > 1) { - leave("i82092aa_set_io_map with invalid map"); + + /* Check error conditions */ + if (map > 1) return -EINVAL; - } - if ((io->start > 0xffff) || (io->stop > 0xffff) || (io->stop < io->start)){ - leave("i82092aa_set_io_map with invalid io"); + + if ((io->start > 0xffff) || (io->stop > 0xffff) + || (io->stop < io->start)) return -EINVAL; - } - /* Turn off the window before changing anything */ + /* Turn off the window before changing anything */ if (indirect_read(sock, I365_ADDRWIN) & I365_ENA_IO(map)) indirect_resetbit(sock, I365_ADDRWIN, I365_ENA_IO(map)); -/* printk("set_io_map: Setting range to %x - %x \n",io->start,io->stop); */ - /* write the new values */ - indirect_write16(sock,I365_IO(map)+I365_W_START,io->start); - indirect_write16(sock,I365_IO(map)+I365_W_STOP,io->stop); - - ioctl = indirect_read(sock,I365_IOCTL) & ~I365_IOCTL_MASK(map); - + indirect_write16(sock, I365_IO(map)+I365_W_START, io->start); + indirect_write16(sock, I365_IO(map)+I365_W_STOP, io->stop); + + ioctl = indirect_read(sock, I365_IOCTL) & ~I365_IOCTL_MASK(map); + if (io->flags & (MAP_16BIT|MAP_AUTOSZ)) ioctl |= I365_IOCTL_16BIT(map); - - indirect_write(sock,I365_IOCTL,ioctl); - + + indirect_write(sock, I365_IOCTL, ioctl); + /* Turn the window back on if needed */ if (io->flags & MAP_ACTIVE) - indirect_setbit(sock,I365_ADDRWIN,I365_ENA_IO(map)); - - leave("i82092aa_set_io_map"); + indirect_setbit(sock, I365_ADDRWIN, I365_ENA_IO(map)); + return 0; } -static int i82092aa_set_mem_map(struct pcmcia_socket *socket, struct pccard_mem_map *mem) +static int i82092aa_set_mem_map(struct pcmcia_socket *socket, + struct pccard_mem_map *mem) { - struct socket_info *sock_info = container_of(socket, struct socket_info, socket); + struct socket_info *sock_info = container_of(socket, struct socket_info, + socket); unsigned int sock = sock_info->number; struct pci_bus_region region; unsigned short base, i; unsigned char map; - - enter("i82092aa_set_mem_map"); pcibios_resource_to_bus(sock_info->dev->bus, ®ion, mem->res); - + map = mem->map; - if (map > 4) { - leave("i82092aa_set_mem_map: invalid map"); + if (map > 4) return -EINVAL; - } - - - if ( (mem->card_start > 0x3ffffff) || (region.start > region.end) || - (mem->speed > 1000) ) { - leave("i82092aa_set_mem_map: invalid address / speed"); - printk("invalid mem map for socket %i: %llx to %llx with a " - "start of %x\n", + + if ((mem->card_start > 0x3ffffff) || (region.start > region.end) || + (mem->speed > 1000)) { + dev_err(&sock_info->dev->dev, + "invalid mem map for socket %i: %llx to %llx with a start of %x\n", sock, (unsigned long long)region.start, (unsigned long long)region.end, mem->card_start); return -EINVAL; } - + /* Turn off the window before changing anything */ if (indirect_read(sock, I365_ADDRWIN) & I365_ENA_MEM(map)) - indirect_resetbit(sock, I365_ADDRWIN, I365_ENA_MEM(map)); - - -/* printk("set_mem_map: Setting map %i range to %x - %x on socket %i, speed is %i, active = %i \n",map, region.start,region.end,sock,mem->speed,mem->flags & MAP_ACTIVE); */ + indirect_resetbit(sock, I365_ADDRWIN, I365_ENA_MEM(map)); /* write the start address */ base = I365_MEM(map); i = (region.start >> 12) & 0x0fff; - if (mem->flags & MAP_16BIT) + if (mem->flags & MAP_16BIT) i |= I365_MEM_16BIT; if (mem->flags & MAP_0WS) - i |= I365_MEM_0WS; - indirect_write16(sock,base+I365_W_START,i); - + i |= I365_MEM_0WS; + indirect_write16(sock, base+I365_W_START, i); + /* write the stop address */ - - i= (region.end >> 12) & 0x0fff; + + i = (region.end >> 12) & 0x0fff; switch (to_cycles(mem->speed)) { - case 0: - break; - case 1: - i |= I365_MEM_WS0; - break; - case 2: - i |= I365_MEM_WS1; - break; - default: - i |= I365_MEM_WS1 | I365_MEM_WS0; - break; + case 0: + break; + case 1: + i |= I365_MEM_WS0; + break; + case 2: + i |= I365_MEM_WS1; + break; + default: + i |= I365_MEM_WS1 | I365_MEM_WS0; + break; } - - indirect_write16(sock,base+I365_W_STOP,i); - + + indirect_write16(sock, base+I365_W_STOP, i); + /* card start */ - + i = ((mem->card_start - region.start) >> 12) & 0x3fff; if (mem->flags & MAP_WRPROT) i |= I365_MEM_WRPROT; - if (mem->flags & MAP_ATTRIB) { -/* printk("requesting attribute memory for socket %i\n",sock);*/ + if (mem->flags & MAP_ATTRIB) i |= I365_MEM_REG; - } else { -/* printk("requesting normal memory for socket %i\n",sock);*/ - } - indirect_write16(sock,base+I365_W_OFF,i); - + indirect_write16(sock, base+I365_W_OFF, i); + /* Enable the window if necessary */ if (mem->flags & MAP_ACTIVE) indirect_setbit(sock, I365_ADDRWIN, I365_ENA_MEM(map)); - - leave("i82092aa_set_mem_map"); + return 0; } @@ -691,11 +667,9 @@ static int i82092aa_module_init(void) static void i82092aa_module_exit(void) { - enter("i82092aa_module_exit"); pci_unregister_driver(&i82092aa_pci_driver); - if (sockets[0].io_base>0) - release_region(sockets[0].io_base, 2); - leave("i82092aa_module_exit"); + if (sockets[0].io_base > 0) + release_region(sockets[0].io_base, 2); } module_init(i82092aa_module_init); diff --git a/drivers/pcmcia/i82092aa.h b/drivers/pcmcia/i82092aa.h index 4586c43c78e2..0f851acab7e5 100644 --- a/drivers/pcmcia/i82092aa.h +++ b/drivers/pcmcia/i82092aa.h @@ -4,17 +4,6 @@ #include <linux/interrupt.h> -/* Debuging defines */ -#ifdef NOTRACE -#define enter(x) printk("Enter: %s, %s line %i\n",x,__FILE__,__LINE__) -#define leave(x) printk("Leave: %s, %s line %i\n",x,__FILE__,__LINE__) -#else -#define enter(x) do {} while (0) -#define leave(x) do {} while (0) -#endif - - - /* prototypes */ static int i82092aa_pci_probe(struct pci_dev *dev, const struct pci_device_id *id); diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c index 2eb28cc2d2dc..cd5a6c95dbdc 100644 --- a/drivers/phy/phy-core.c +++ b/drivers/phy/phy-core.c @@ -712,7 +712,7 @@ struct phy *phy_optional_get(struct device *dev, const char *string) { struct phy *phy = phy_get(dev, string); - if (IS_ERR(phy) && (PTR_ERR(phy) == -ENODEV)) + if (PTR_ERR(phy) == -ENODEV) phy = NULL; return phy; @@ -766,7 +766,7 @@ struct phy *devm_phy_optional_get(struct device *dev, const char *string) { struct phy *phy = devm_phy_get(dev, string); - if (IS_ERR(phy) && (PTR_ERR(phy) == -ENODEV)) + if (PTR_ERR(phy) == -ENODEV) phy = NULL; return phy; diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c index 21c370dbbfba..bddf2c5dd3bf 100644 --- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c +++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c @@ -10,6 +10,7 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/module.h> +#include <linux/pinctrl/machine.h> #include <linux/pinctrl/pinconf.h> #include <linux/pinctrl/pinconf-generic.h> #include <linux/pinctrl/pinmux.h> diff --git a/drivers/platform/chrome/chromeos_laptop.c b/drivers/platform/chrome/chromeos_laptop.c index 8723bcf10c93..4f3651fcd9fe 100644 --- a/drivers/platform/chrome/chromeos_laptop.c +++ b/drivers/platform/chrome/chromeos_laptop.c @@ -63,7 +63,7 @@ struct acpi_peripheral { struct chromeos_laptop { /* * Note that we can't mark this pointer as const because - * i2c_new_probed_device() changes passed in I2C board info, so. + * i2c_new_scanned_device() changes passed in I2C board info, so. */ struct i2c_peripheral *i2c_peripherals; unsigned int num_i2c_peripherals; @@ -87,8 +87,8 @@ chromes_laptop_instantiate_i2c_device(struct i2c_adapter *adapter, * address we scan secondary addresses. In any case the client * structure gets assigned primary address. */ - client = i2c_new_probed_device(adapter, info, addr_list, NULL); - if (!client && alt_addr) { + client = i2c_new_scanned_device(adapter, info, addr_list, NULL); + if (IS_ERR(client) && alt_addr) { struct i2c_board_info dummy_info = { I2C_BOARD_INFO("dummy", info->addr), }; @@ -97,9 +97,9 @@ chromes_laptop_instantiate_i2c_device(struct i2c_adapter *adapter, }; struct i2c_client *dummy; - dummy = i2c_new_probed_device(adapter, &dummy_info, - alt_addr_list, NULL); - if (dummy) { + dummy = i2c_new_scanned_device(adapter, &dummy_info, + alt_addr_list, NULL); + if (!IS_ERR(dummy)) { pr_debug("%d-%02x is probed at %02x\n", adapter->nr, info->addr, dummy->addr); i2c_unregister_device(dummy); @@ -107,12 +107,14 @@ chromes_laptop_instantiate_i2c_device(struct i2c_adapter *adapter, } } - if (!client) + if (IS_ERR(client)) { + client = NULL; pr_debug("failed to register device %d-%02x\n", adapter->nr, info->addr); - else + } else { pr_debug("added i2c device %d-%02x\n", adapter->nr, info->addr); + } return client; } diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 6d6ce86a1408..6fc8f2c3ac51 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -16,7 +16,8 @@ #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> #include <linux/suspend.h> -#include <asm/unaligned.h> + +#include "cros_ec.h" #define CROS_EC_DEV_EC_INDEX 0 #define CROS_EC_DEV_PD_INDEX 1 diff --git a/drivers/platform/chrome/cros_ec.h b/drivers/platform/chrome/cros_ec.h new file mode 100644 index 000000000000..e69fc1ff68b4 --- /dev/null +++ b/drivers/platform/chrome/cros_ec.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ChromeOS Embedded Controller core interface. + * + * Copyright (C) 2020 Google LLC + */ + +#ifndef __CROS_EC_H +#define __CROS_EC_H + +int cros_ec_register(struct cros_ec_device *ec_dev); +int cros_ec_unregister(struct cros_ec_device *ec_dev); + +int cros_ec_suspend(struct cros_ec_device *ec_dev); +int cros_ec_resume(struct cros_ec_device *ec_dev); + +bool cros_ec_handle_event(struct cros_ec_device *ec_dev); + +#endif /* __CROS_EC_H */ diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c index 74ded441bb50..c65e70bc168d 100644 --- a/drivers/platform/chrome/cros_ec_chardev.c +++ b/drivers/platform/chrome/cros_ec_chardev.c @@ -13,7 +13,6 @@ #include <linux/init.h> #include <linux/device.h> #include <linux/fs.h> -#include <linux/mfd/cros_ec.h> #include <linux/miscdevice.h> #include <linux/module.h> #include <linux/notifier.h> diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c index 6ae484989d1f..ecfada00e6c5 100644 --- a/drivers/platform/chrome/cros_ec_debugfs.c +++ b/drivers/platform/chrome/cros_ec_debugfs.c @@ -7,7 +7,6 @@ #include <linux/debugfs.h> #include <linux/delay.h> #include <linux/fs.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/platform_data/cros_ec_commands.h> diff --git a/drivers/platform/chrome/cros_ec_i2c.c b/drivers/platform/chrome/cros_ec_i2c.c index 9bd97bc8454b..6119eccd8a18 100644 --- a/drivers/platform/chrome/cros_ec_i2c.c +++ b/drivers/platform/chrome/cros_ec_i2c.c @@ -14,6 +14,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> +#include "cros_ec.h" + /** * Request format for protocol v3 * byte 0 0xda (EC_COMMAND_PROTOCOL_3) diff --git a/drivers/platform/chrome/cros_ec_ishtp.c b/drivers/platform/chrome/cros_ec_ishtp.c index e5996821d08b..93a71e93a2f1 100644 --- a/drivers/platform/chrome/cros_ec_ishtp.c +++ b/drivers/platform/chrome/cros_ec_ishtp.c @@ -14,6 +14,8 @@ #include <linux/platform_data/cros_ec_proto.h> #include <linux/intel-ish-client-if.h> +#include "cros_ec.h" + /* * ISH TX/RX ring buffer pool size * @@ -76,7 +78,7 @@ struct cros_ish_in_msg { * * The writers are .reset() and .probe() function. */ -DECLARE_RWSEM(init_lock); +static DECLARE_RWSEM(init_lock); /** * struct response_info - Encapsulate firmware response related diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index c0f2eec35a48..b4c110c5fee0 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -8,7 +8,6 @@ #include <linux/device.h> #include <linux/fs.h> #include <linux/kobject.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> diff --git a/drivers/platform/chrome/cros_ec_lpc.c b/drivers/platform/chrome/cros_ec_lpc.c index dccf479c6625..1f7861944044 100644 --- a/drivers/platform/chrome/cros_ec_lpc.c +++ b/drivers/platform/chrome/cros_ec_lpc.c @@ -23,6 +23,7 @@ #include <linux/printk.h> #include <linux/suspend.h> +#include "cros_ec.h" #include "cros_ec_lpc_mec.h" #define DRV_NAME "cros_ec_lpcs" @@ -396,7 +397,7 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) * Some boards do not have an IRQ allotted for cros_ec_lpc, * which makes ENXIO an expected (and safe) scenario. */ - irq = platform_get_irq(pdev, 0); + irq = platform_get_irq_optional(pdev, 0); if (irq > 0) ec_dev->irq = irq; else if (irq != -ENXIO) { diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index da1b1c450433..3cfa643f1d07 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -54,8 +54,6 @@ static int send_command(struct cros_ec_device *ec_dev, int ret; int (*xfer_fxn)(struct cros_ec_device *ec, struct cros_ec_command *msg); - trace_cros_ec_cmd(msg); - if (ec_dev->proto_version > 2) xfer_fxn = ec_dev->pkt_xfer; else @@ -72,7 +70,9 @@ static int send_command(struct cros_ec_device *ec_dev, return -EIO; } + trace_cros_ec_request_start(msg); ret = (*xfer_fxn)(ec_dev, msg); + trace_cros_ec_request_done(msg, ret); if (msg->result == EC_RES_IN_PROGRESS) { int i; struct cros_ec_command *status_msg; @@ -95,7 +95,9 @@ static int send_command(struct cros_ec_device *ec_dev, for (i = 0; i < EC_COMMAND_RETRIES; i++) { usleep_range(10000, 11000); + trace_cros_ec_request_start(status_msg); ret = (*xfer_fxn)(ec_dev, status_msg); + trace_cros_ec_request_done(status_msg, ret); if (ret == -EAGAIN) continue; if (ret < 0) diff --git a/drivers/platform/chrome/cros_ec_rpmsg.c b/drivers/platform/chrome/cros_ec_rpmsg.c index bd068afe43b5..dbc3f5523b83 100644 --- a/drivers/platform/chrome/cros_ec_rpmsg.c +++ b/drivers/platform/chrome/cros_ec_rpmsg.c @@ -13,6 +13,8 @@ #include <linux/rpmsg.h> #include <linux/slab.h> +#include "cros_ec.h" + #define EC_MSG_TIMEOUT_MS 200 #define HOST_COMMAND_MARK 1 #define HOST_EVENT_MARK 2 diff --git a/drivers/platform/chrome/cros_ec_sensorhub.c b/drivers/platform/chrome/cros_ec_sensorhub.c index 04d8879689e9..79fefd3bb0fa 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub.c +++ b/drivers/platform/chrome/cros_ec_sensorhub.c @@ -9,7 +9,6 @@ #include <linux/init.h> #include <linux/device.h> #include <linux/module.h> -#include <linux/mfd/cros_ec.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> #include <linux/platform_data/cros_ec_sensorhub.h> diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index a831bd5a5b2f..46786d2d679a 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -14,6 +14,8 @@ #include <linux/spi/spi.h> #include <uapi/linux/sched/types.h> +#include "cros_ec.h" + /* The header byte, which follows the preamble */ #define EC_MSG_HEADER 0xec diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c index 74d36b8d4f46..07dac97ad57c 100644 --- a/drivers/platform/chrome/cros_ec_sysfs.c +++ b/drivers/platform/chrome/cros_ec_sysfs.c @@ -8,7 +8,6 @@ #include <linux/device.h> #include <linux/fs.h> #include <linux/kobject.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> diff --git a/drivers/platform/chrome/cros_ec_trace.c b/drivers/platform/chrome/cros_ec_trace.c index 5af1d66d9eca..523a39bd0ff6 100644 --- a/drivers/platform/chrome/cros_ec_trace.c +++ b/drivers/platform/chrome/cros_ec_trace.c @@ -8,6 +8,11 @@ // Generate the list using the following script: // sed -n 's/^#define \(EC_CMD_[[:alnum:]_]*\)\s.*/\tTRACE_SYMBOL(\1), \\/p' include/linux/platform_data/cros_ec_commands.h #define EC_CMDS \ + TRACE_SYMBOL(EC_CMD_ACPI_READ), \ + TRACE_SYMBOL(EC_CMD_ACPI_WRITE), \ + TRACE_SYMBOL(EC_CMD_ACPI_BURST_ENABLE), \ + TRACE_SYMBOL(EC_CMD_ACPI_BURST_DISABLE), \ + TRACE_SYMBOL(EC_CMD_ACPI_QUERY_EVENT), \ TRACE_SYMBOL(EC_CMD_PROTO_VERSION), \ TRACE_SYMBOL(EC_CMD_HELLO), \ TRACE_SYMBOL(EC_CMD_GET_VERSION), \ @@ -22,6 +27,8 @@ TRACE_SYMBOL(EC_CMD_GET_PROTOCOL_INFO), \ TRACE_SYMBOL(EC_CMD_GSV_PAUSE_IN_S5), \ TRACE_SYMBOL(EC_CMD_GET_FEATURES), \ + TRACE_SYMBOL(EC_CMD_GET_SKU_ID), \ + TRACE_SYMBOL(EC_CMD_SET_SKU_ID), \ TRACE_SYMBOL(EC_CMD_FLASH_INFO), \ TRACE_SYMBOL(EC_CMD_FLASH_READ), \ TRACE_SYMBOL(EC_CMD_FLASH_WRITE), \ @@ -29,6 +36,8 @@ TRACE_SYMBOL(EC_CMD_FLASH_PROTECT), \ TRACE_SYMBOL(EC_CMD_FLASH_REGION_INFO), \ TRACE_SYMBOL(EC_CMD_VBNV_CONTEXT), \ + TRACE_SYMBOL(EC_CMD_FLASH_SPI_INFO), \ + TRACE_SYMBOL(EC_CMD_FLASH_SELECT), \ TRACE_SYMBOL(EC_CMD_PWM_GET_FAN_TARGET_RPM), \ TRACE_SYMBOL(EC_CMD_PWM_SET_FAN_TARGET_RPM), \ TRACE_SYMBOL(EC_CMD_PWM_GET_KEYBOARD_BACKLIGHT), \ @@ -40,6 +49,8 @@ TRACE_SYMBOL(EC_CMD_LED_CONTROL), \ TRACE_SYMBOL(EC_CMD_VBOOT_HASH), \ TRACE_SYMBOL(EC_CMD_MOTION_SENSE_CMD), \ + TRACE_SYMBOL(EC_CMD_FORCE_LID_OPEN), \ + TRACE_SYMBOL(EC_CMD_CONFIG_POWER_BUTTON), \ TRACE_SYMBOL(EC_CMD_USB_CHARGE_SET_MODE), \ TRACE_SYMBOL(EC_CMD_PSTORE_INFO), \ TRACE_SYMBOL(EC_CMD_PSTORE_READ), \ @@ -50,6 +61,9 @@ TRACE_SYMBOL(EC_CMD_RTC_SET_ALARM), \ TRACE_SYMBOL(EC_CMD_PORT80_LAST_BOOT), \ TRACE_SYMBOL(EC_CMD_PORT80_READ), \ + TRACE_SYMBOL(EC_CMD_VSTORE_INFO), \ + TRACE_SYMBOL(EC_CMD_VSTORE_READ), \ + TRACE_SYMBOL(EC_CMD_VSTORE_WRITE), \ TRACE_SYMBOL(EC_CMD_THERMAL_SET_THRESHOLD), \ TRACE_SYMBOL(EC_CMD_THERMAL_GET_THRESHOLD), \ TRACE_SYMBOL(EC_CMD_THERMAL_AUTO_FAN_CTRL), \ @@ -59,10 +73,12 @@ TRACE_SYMBOL(EC_CMD_MKBP_STATE), \ TRACE_SYMBOL(EC_CMD_MKBP_INFO), \ TRACE_SYMBOL(EC_CMD_MKBP_SIMULATE_KEY), \ + TRACE_SYMBOL(EC_CMD_GET_KEYBOARD_ID), \ TRACE_SYMBOL(EC_CMD_MKBP_SET_CONFIG), \ TRACE_SYMBOL(EC_CMD_MKBP_GET_CONFIG), \ TRACE_SYMBOL(EC_CMD_KEYSCAN_SEQ_CTRL), \ TRACE_SYMBOL(EC_CMD_GET_NEXT_EVENT), \ + TRACE_SYMBOL(EC_CMD_KEYBOARD_FACTORY_TEST), \ TRACE_SYMBOL(EC_CMD_TEMP_SENSOR_GET_INFO), \ TRACE_SYMBOL(EC_CMD_HOST_EVENT_GET_B), \ TRACE_SYMBOL(EC_CMD_HOST_EVENT_GET_SMI_MASK), \ @@ -73,6 +89,7 @@ TRACE_SYMBOL(EC_CMD_HOST_EVENT_CLEAR), \ TRACE_SYMBOL(EC_CMD_HOST_EVENT_SET_WAKE_MASK), \ TRACE_SYMBOL(EC_CMD_HOST_EVENT_CLEAR_B), \ + TRACE_SYMBOL(EC_CMD_HOST_EVENT), \ TRACE_SYMBOL(EC_CMD_SWITCH_ENABLE_BKLIGHT), \ TRACE_SYMBOL(EC_CMD_SWITCH_ENABLE_WIRELESS), \ TRACE_SYMBOL(EC_CMD_GPIO_SET), \ @@ -92,36 +109,102 @@ TRACE_SYMBOL(EC_CMD_CHARGE_STATE), \ TRACE_SYMBOL(EC_CMD_CHARGE_CURRENT_LIMIT), \ TRACE_SYMBOL(EC_CMD_EXTERNAL_POWER_LIMIT), \ + TRACE_SYMBOL(EC_CMD_OVERRIDE_DEDICATED_CHARGER_LIMIT), \ + TRACE_SYMBOL(EC_CMD_HIBERNATION_DELAY), \ TRACE_SYMBOL(EC_CMD_HOST_SLEEP_EVENT), \ + TRACE_SYMBOL(EC_CMD_DEVICE_EVENT), \ TRACE_SYMBOL(EC_CMD_SB_READ_WORD), \ TRACE_SYMBOL(EC_CMD_SB_WRITE_WORD), \ TRACE_SYMBOL(EC_CMD_SB_READ_BLOCK), \ TRACE_SYMBOL(EC_CMD_SB_WRITE_BLOCK), \ TRACE_SYMBOL(EC_CMD_BATTERY_VENDOR_PARAM), \ + TRACE_SYMBOL(EC_CMD_SB_FW_UPDATE), \ + TRACE_SYMBOL(EC_CMD_ENTERING_MODE), \ + TRACE_SYMBOL(EC_CMD_I2C_PASSTHRU_PROTECT), \ + TRACE_SYMBOL(EC_CMD_CEC_WRITE_MSG), \ + TRACE_SYMBOL(EC_CMD_CEC_SET), \ + TRACE_SYMBOL(EC_CMD_CEC_GET), \ TRACE_SYMBOL(EC_CMD_EC_CODEC), \ TRACE_SYMBOL(EC_CMD_EC_CODEC_DMIC), \ TRACE_SYMBOL(EC_CMD_EC_CODEC_I2S_RX), \ TRACE_SYMBOL(EC_CMD_EC_CODEC_WOV), \ TRACE_SYMBOL(EC_CMD_REBOOT_EC), \ TRACE_SYMBOL(EC_CMD_GET_PANIC_INFO), \ - TRACE_SYMBOL(EC_CMD_ACPI_READ), \ - TRACE_SYMBOL(EC_CMD_ACPI_WRITE), \ - TRACE_SYMBOL(EC_CMD_ACPI_QUERY_EVENT), \ - TRACE_SYMBOL(EC_CMD_CEC_WRITE_MSG), \ - TRACE_SYMBOL(EC_CMD_CEC_SET), \ - TRACE_SYMBOL(EC_CMD_CEC_GET), \ TRACE_SYMBOL(EC_CMD_REBOOT), \ TRACE_SYMBOL(EC_CMD_RESEND_RESPONSE), \ TRACE_SYMBOL(EC_CMD_VERSION0), \ TRACE_SYMBOL(EC_CMD_PD_EXCHANGE_STATUS), \ + TRACE_SYMBOL(EC_CMD_PD_HOST_EVENT_STATUS), \ TRACE_SYMBOL(EC_CMD_USB_PD_CONTROL), \ TRACE_SYMBOL(EC_CMD_USB_PD_PORTS), \ TRACE_SYMBOL(EC_CMD_USB_PD_POWER_INFO), \ TRACE_SYMBOL(EC_CMD_CHARGE_PORT_COUNT), \ + TRACE_SYMBOL(EC_CMD_USB_PD_FW_UPDATE), \ + TRACE_SYMBOL(EC_CMD_USB_PD_RW_HASH_ENTRY), \ + TRACE_SYMBOL(EC_CMD_USB_PD_DEV_INFO), \ TRACE_SYMBOL(EC_CMD_USB_PD_DISCOVERY), \ TRACE_SYMBOL(EC_CMD_PD_CHARGE_PORT_OVERRIDE), \ TRACE_SYMBOL(EC_CMD_PD_GET_LOG_ENTRY), \ - TRACE_SYMBOL(EC_CMD_USB_PD_MUX_INFO) + TRACE_SYMBOL(EC_CMD_USB_PD_GET_AMODE), \ + TRACE_SYMBOL(EC_CMD_USB_PD_SET_AMODE), \ + TRACE_SYMBOL(EC_CMD_PD_WRITE_LOG_ENTRY), \ + TRACE_SYMBOL(EC_CMD_PD_CONTROL), \ + TRACE_SYMBOL(EC_CMD_USB_PD_MUX_INFO), \ + TRACE_SYMBOL(EC_CMD_PD_CHIP_INFO), \ + TRACE_SYMBOL(EC_CMD_RWSIG_CHECK_STATUS), \ + TRACE_SYMBOL(EC_CMD_RWSIG_ACTION), \ + TRACE_SYMBOL(EC_CMD_EFS_VERIFY), \ + TRACE_SYMBOL(EC_CMD_GET_CROS_BOARD_INFO), \ + TRACE_SYMBOL(EC_CMD_SET_CROS_BOARD_INFO), \ + TRACE_SYMBOL(EC_CMD_GET_UPTIME_INFO), \ + TRACE_SYMBOL(EC_CMD_ADD_ENTROPY), \ + TRACE_SYMBOL(EC_CMD_ADC_READ), \ + TRACE_SYMBOL(EC_CMD_ROLLBACK_INFO), \ + TRACE_SYMBOL(EC_CMD_AP_RESET), \ + TRACE_SYMBOL(EC_CMD_CR51_BASE), \ + TRACE_SYMBOL(EC_CMD_CR51_LAST), \ + TRACE_SYMBOL(EC_CMD_FP_PASSTHRU), \ + TRACE_SYMBOL(EC_CMD_FP_MODE), \ + TRACE_SYMBOL(EC_CMD_FP_INFO), \ + TRACE_SYMBOL(EC_CMD_FP_FRAME), \ + TRACE_SYMBOL(EC_CMD_FP_TEMPLATE), \ + TRACE_SYMBOL(EC_CMD_FP_CONTEXT), \ + TRACE_SYMBOL(EC_CMD_FP_STATS), \ + TRACE_SYMBOL(EC_CMD_FP_SEED), \ + TRACE_SYMBOL(EC_CMD_FP_ENC_STATUS), \ + TRACE_SYMBOL(EC_CMD_TP_SELF_TEST), \ + TRACE_SYMBOL(EC_CMD_TP_FRAME_INFO), \ + TRACE_SYMBOL(EC_CMD_TP_FRAME_SNAPSHOT), \ + TRACE_SYMBOL(EC_CMD_TP_FRAME_GET), \ + TRACE_SYMBOL(EC_CMD_BATTERY_GET_STATIC), \ + TRACE_SYMBOL(EC_CMD_BATTERY_GET_DYNAMIC), \ + TRACE_SYMBOL(EC_CMD_CHARGER_CONTROL), \ + TRACE_SYMBOL(EC_CMD_BOARD_SPECIFIC_BASE), \ + TRACE_SYMBOL(EC_CMD_BOARD_SPECIFIC_LAST) + +/* See the enum ec_status in include/linux/platform_data/cros_ec_commands.h */ +#define EC_RESULT \ + TRACE_SYMBOL(EC_RES_SUCCESS), \ + TRACE_SYMBOL(EC_RES_INVALID_COMMAND), \ + TRACE_SYMBOL(EC_RES_ERROR), \ + TRACE_SYMBOL(EC_RES_INVALID_PARAM), \ + TRACE_SYMBOL(EC_RES_ACCESS_DENIED), \ + TRACE_SYMBOL(EC_RES_INVALID_RESPONSE), \ + TRACE_SYMBOL(EC_RES_INVALID_VERSION), \ + TRACE_SYMBOL(EC_RES_INVALID_CHECKSUM), \ + TRACE_SYMBOL(EC_RES_IN_PROGRESS), \ + TRACE_SYMBOL(EC_RES_UNAVAILABLE), \ + TRACE_SYMBOL(EC_RES_TIMEOUT), \ + TRACE_SYMBOL(EC_RES_OVERFLOW), \ + TRACE_SYMBOL(EC_RES_INVALID_HEADER), \ + TRACE_SYMBOL(EC_RES_REQUEST_TRUNCATED), \ + TRACE_SYMBOL(EC_RES_RESPONSE_TOO_BIG), \ + TRACE_SYMBOL(EC_RES_BUS_ERROR), \ + TRACE_SYMBOL(EC_RES_BUSY), \ + TRACE_SYMBOL(EC_RES_INVALID_HEADER_VERSION), \ + TRACE_SYMBOL(EC_RES_INVALID_HEADER_CRC), \ + TRACE_SYMBOL(EC_RES_INVALID_DATA_CRC), \ + TRACE_SYMBOL(EC_RES_DUP_UNAVAILABLE) #define CREATE_TRACE_POINTS #include "cros_ec_trace.h" diff --git a/drivers/platform/chrome/cros_ec_trace.h b/drivers/platform/chrome/cros_ec_trace.h index 0dd4df30fa89..e9fb05f89ef0 100644 --- a/drivers/platform/chrome/cros_ec_trace.h +++ b/drivers/platform/chrome/cros_ec_trace.h @@ -18,7 +18,7 @@ #include <linux/tracepoint.h> -DECLARE_EVENT_CLASS(cros_ec_cmd_class, +TRACE_EVENT(cros_ec_request_start, TP_PROTO(struct cros_ec_command *cmd), TP_ARGS(cmd), TP_STRUCT__entry( @@ -33,10 +33,26 @@ DECLARE_EVENT_CLASS(cros_ec_cmd_class, __print_symbolic(__entry->command, EC_CMDS)) ); - -DEFINE_EVENT(cros_ec_cmd_class, cros_ec_cmd, - TP_PROTO(struct cros_ec_command *cmd), - TP_ARGS(cmd) +TRACE_EVENT(cros_ec_request_done, + TP_PROTO(struct cros_ec_command *cmd, int retval), + TP_ARGS(cmd, retval), + TP_STRUCT__entry( + __field(uint32_t, version) + __field(uint32_t, command) + __field(uint32_t, result) + __field(int, retval) + ), + TP_fast_assign( + __entry->version = cmd->version; + __entry->command = cmd->command; + __entry->result = cmd->result; + __entry->retval = retval; + ), + TP_printk("version: %u, command: %s, ec result: %s, retval: %d", + __entry->version, + __print_symbolic(__entry->command, EC_CMDS), + __print_symbolic(__entry->result, EC_RESULT), + __entry->retval) ); diff --git a/drivers/platform/chrome/cros_ec_vbc.c b/drivers/platform/chrome/cros_ec_vbc.c index f11a1283e5c8..8edae465105c 100644 --- a/drivers/platform/chrome/cros_ec_vbc.c +++ b/drivers/platform/chrome/cros_ec_vbc.c @@ -6,7 +6,6 @@ #include <linux/of.h> #include <linux/platform_device.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> diff --git a/drivers/platform/chrome/cros_usbpd_logger.c b/drivers/platform/chrome/cros_usbpd_logger.c index 374cdd1e868a..7de3ea75ef46 100644 --- a/drivers/platform/chrome/cros_usbpd_logger.c +++ b/drivers/platform/chrome/cros_usbpd_logger.c @@ -6,7 +6,6 @@ */ #include <linux/ktime.h> -#include <linux/mfd/cros_ec.h> #include <linux/math64.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> diff --git a/drivers/platform/chrome/wilco_ec/Kconfig b/drivers/platform/chrome/wilco_ec/Kconfig index 365f30e116ee..49e8530ca0ac 100644 --- a/drivers/platform/chrome/wilco_ec/Kconfig +++ b/drivers/platform/chrome/wilco_ec/Kconfig @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only config WILCO_EC tristate "ChromeOS Wilco Embedded Controller" - depends on ACPI && X86 && CROS_EC_LPC && LEDS_CLASS + depends on X86 || COMPILE_TEST + depends on ACPI && CROS_EC_LPC && LEDS_CLASS help If you say Y here, you get support for talking to the ChromeOS Wilco EC over an eSPI bus. This uses a simple byte-level protocol diff --git a/drivers/platform/chrome/wilco_ec/core.c b/drivers/platform/chrome/wilco_ec/core.c index 5210c357feef..5b42992bff38 100644 --- a/drivers/platform/chrome/wilco_ec/core.c +++ b/drivers/platform/chrome/wilco_ec/core.c @@ -94,7 +94,7 @@ static int wilco_ec_probe(struct platform_device *pdev) ret = wilco_ec_add_sysfs(ec); if (ret < 0) { - dev_err(dev, "Failed to create sysfs entries: %d", ret); + dev_err(dev, "Failed to create sysfs entries: %d\n", ret); goto unregister_rtc; } @@ -137,9 +137,9 @@ static int wilco_ec_remove(struct platform_device *pdev) { struct wilco_ec_device *ec = platform_get_drvdata(pdev); + platform_device_unregister(ec->telem_pdev); platform_device_unregister(ec->charger_pdev); wilco_ec_remove_sysfs(ec); - platform_device_unregister(ec->telem_pdev); platform_device_unregister(ec->rtc_pdev); if (ec->debugfs_pdev) platform_device_unregister(ec->debugfs_pdev); diff --git a/drivers/platform/chrome/wilco_ec/keyboard_leds.c b/drivers/platform/chrome/wilco_ec/keyboard_leds.c index 5731d1b60e28..6ce9c6782065 100644 --- a/drivers/platform/chrome/wilco_ec/keyboard_leds.c +++ b/drivers/platform/chrome/wilco_ec/keyboard_leds.c @@ -69,7 +69,7 @@ static int send_kbbl_msg(struct wilco_ec_device *ec, ret = wilco_ec_mailbox(ec, &msg); if (ret < 0) { dev_err(ec->dev, - "Failed sending keyboard LEDs command: %d", ret); + "Failed sending keyboard LEDs command: %d\n", ret); return ret; } @@ -94,7 +94,7 @@ static int set_kbbl(struct wilco_ec_device *ec, enum led_brightness brightness) if (response.status) { dev_err(ec->dev, - "EC reported failure sending keyboard LEDs command: %d", + "EC reported failure sending keyboard LEDs command: %d\n", response.status); return -EIO; } @@ -147,7 +147,7 @@ static int kbbl_init(struct wilco_ec_device *ec) if (response.status) { dev_err(ec->dev, - "EC reported failure sending keyboard LEDs command: %d", + "EC reported failure sending keyboard LEDs command: %d\n", response.status); return -EIO; } @@ -179,7 +179,7 @@ int wilco_keyboard_leds_init(struct wilco_ec_device *ec) ret = kbbl_exist(ec, &leds_exist); if (ret < 0) { dev_err(ec->dev, - "Failed checking keyboard LEDs support: %d", ret); + "Failed checking keyboard LEDs support: %d\n", ret); return ret; } if (!leds_exist) diff --git a/drivers/platform/chrome/wilco_ec/mailbox.c b/drivers/platform/chrome/wilco_ec/mailbox.c index ced1f9f3dcee..0f98358ea824 100644 --- a/drivers/platform/chrome/wilco_ec/mailbox.c +++ b/drivers/platform/chrome/wilco_ec/mailbox.c @@ -163,13 +163,13 @@ static int wilco_ec_transfer(struct wilco_ec_device *ec, } if (rs->data_size != EC_MAILBOX_DATA_SIZE) { - dev_dbg(ec->dev, "unexpected packet size (%u != %u)", + dev_dbg(ec->dev, "unexpected packet size (%u != %u)\n", rs->data_size, EC_MAILBOX_DATA_SIZE); return -EMSGSIZE; } if (rs->data_size < msg->response_size) { - dev_dbg(ec->dev, "EC didn't return enough data (%u < %zu)", + dev_dbg(ec->dev, "EC didn't return enough data (%u < %zu)\n", rs->data_size, msg->response_size); return -EMSGSIZE; } diff --git a/drivers/platform/chrome/wilco_ec/telemetry.c b/drivers/platform/chrome/wilco_ec/telemetry.c index 1176d543191a..e06d96fb9426 100644 --- a/drivers/platform/chrome/wilco_ec/telemetry.c +++ b/drivers/platform/chrome/wilco_ec/telemetry.c @@ -367,7 +367,7 @@ static int telem_device_probe(struct platform_device *pdev) minor = ida_alloc_max(&telem_ida, TELEM_MAX_DEV-1, GFP_KERNEL); if (minor < 0) { error = minor; - dev_err(&pdev->dev, "Failed to find minor number: %d", error); + dev_err(&pdev->dev, "Failed to find minor number: %d\n", error); return error; } @@ -427,14 +427,14 @@ static int __init telem_module_init(void) ret = class_register(&telem_class); if (ret) { - pr_err(DRV_NAME ": Failed registering class: %d", ret); + pr_err(DRV_NAME ": Failed registering class: %d\n", ret); return ret; } /* Request the kernel for device numbers, starting with minor=0 */ ret = alloc_chrdev_region(&dev_num, 0, TELEM_MAX_DEV, TELEM_DEV_NAME); if (ret) { - pr_err(DRV_NAME ": Failed allocating dev numbers: %d", ret); + pr_err(DRV_NAME ": Failed allocating dev numbers: %d\n", ret); goto destroy_class; } telem_major = MAJOR(dev_num); diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c index da794dcfdd92..8eaadbaf8ffa 100644 --- a/drivers/platform/x86/thinkpad_acpi.c +++ b/drivers/platform/x86/thinkpad_acpi.c @@ -907,13 +907,12 @@ static ssize_t dispatch_proc_write(struct file *file, return ret; } -static const struct file_operations dispatch_proc_fops = { - .owner = THIS_MODULE, - .open = dispatch_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = dispatch_proc_write, +static const struct proc_ops dispatch_proc_ops = { + .proc_open = dispatch_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = dispatch_proc_write, }; static char *next_cmd(char **cmds) @@ -9984,7 +9983,7 @@ static int __init ibm_init(struct ibm_init_struct *iibm) if (ibm->write) mode |= S_IWUSR; entry = proc_create_data(ibm->name, mode, proc_dir, - &dispatch_proc_fops, ibm); + &dispatch_proc_ops, ibm); if (!entry) { pr_err("unable to create proc entry %s\n", ibm->name); ret = -ENODEV; diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c index a1e6569427c3..808944546739 100644 --- a/drivers/platform/x86/toshiba_acpi.c +++ b/drivers/platform/x86/toshiba_acpi.c @@ -1432,13 +1432,12 @@ static ssize_t lcd_proc_write(struct file *file, const char __user *buf, return count; } -static const struct file_operations lcd_proc_fops = { - .owner = THIS_MODULE, - .open = lcd_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = lcd_proc_write, +static const struct proc_ops lcd_proc_ops = { + .proc_open = lcd_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = lcd_proc_write, }; /* Video-Out */ @@ -1539,13 +1538,12 @@ static ssize_t video_proc_write(struct file *file, const char __user *buf, return ret ? -EIO : count; } -static const struct file_operations video_proc_fops = { - .owner = THIS_MODULE, - .open = video_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = video_proc_write, +static const struct proc_ops video_proc_ops = { + .proc_open = video_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = video_proc_write, }; /* Fan status */ @@ -1617,13 +1615,12 @@ static ssize_t fan_proc_write(struct file *file, const char __user *buf, return count; } -static const struct file_operations fan_proc_fops = { - .owner = THIS_MODULE, - .open = fan_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = fan_proc_write, +static const struct proc_ops fan_proc_ops = { + .proc_open = fan_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = fan_proc_write, }; static int keys_proc_show(struct seq_file *m, void *v) @@ -1662,13 +1659,12 @@ static ssize_t keys_proc_write(struct file *file, const char __user *buf, return count; } -static const struct file_operations keys_proc_fops = { - .owner = THIS_MODULE, - .open = keys_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = keys_proc_write, +static const struct proc_ops keys_proc_ops = { + .proc_open = keys_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = keys_proc_write, }; static int __maybe_unused version_proc_show(struct seq_file *m, void *v) @@ -1688,16 +1684,16 @@ static void create_toshiba_proc_entries(struct toshiba_acpi_dev *dev) { if (dev->backlight_dev) proc_create_data("lcd", S_IRUGO | S_IWUSR, toshiba_proc_dir, - &lcd_proc_fops, dev); + &lcd_proc_ops, dev); if (dev->video_supported) proc_create_data("video", S_IRUGO | S_IWUSR, toshiba_proc_dir, - &video_proc_fops, dev); + &video_proc_ops, dev); if (dev->fan_supported) proc_create_data("fan", S_IRUGO | S_IWUSR, toshiba_proc_dir, - &fan_proc_fops, dev); + &fan_proc_ops, dev); if (dev->hotkey_dev) proc_create_data("keys", S_IRUGO | S_IWUSR, toshiba_proc_dir, - &keys_proc_fops, dev); + &keys_proc_ops, dev); proc_create_single_data("version", S_IRUGO, toshiba_proc_dir, version_proc_show, dev); } diff --git a/drivers/pnp/isapnp/proc.c b/drivers/pnp/isapnp/proc.c index 36820979bd1c..785a796430fa 100644 --- a/drivers/pnp/isapnp/proc.c +++ b/drivers/pnp/isapnp/proc.c @@ -49,10 +49,9 @@ static ssize_t isapnp_proc_bus_read(struct file *file, char __user * buf, return nbytes; } -static const struct file_operations isapnp_proc_bus_file_operations = { - .owner = THIS_MODULE, - .llseek = isapnp_proc_bus_lseek, - .read = isapnp_proc_bus_read, +static const struct proc_ops isapnp_proc_bus_proc_ops = { + .proc_lseek = isapnp_proc_bus_lseek, + .proc_read = isapnp_proc_bus_read, }; static int isapnp_proc_attach_device(struct pnp_dev *dev) @@ -69,7 +68,7 @@ static int isapnp_proc_attach_device(struct pnp_dev *dev) } sprintf(name, "%02x", dev->number); e = dev->procent = proc_create_data(name, S_IFREG | S_IRUGO, de, - &isapnp_proc_bus_file_operations, dev); + &isapnp_proc_bus_proc_ops, dev); if (!e) return -ENOMEM; proc_set_size(e, 256); diff --git a/drivers/pnp/pnpbios/proc.c b/drivers/pnp/pnpbios/proc.c index fe1c8f5d9af0..a806830e3a40 100644 --- a/drivers/pnp/pnpbios/proc.c +++ b/drivers/pnp/pnpbios/proc.c @@ -210,13 +210,12 @@ out: return ret; } -static const struct file_operations pnpbios_proc_fops = { - .owner = THIS_MODULE, - .open = pnpbios_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = pnpbios_proc_write, +static const struct proc_ops pnpbios_proc_ops = { + .proc_open = pnpbios_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = pnpbios_proc_write, }; int pnpbios_interface_attach_device(struct pnp_bios_node *node) @@ -228,13 +227,13 @@ int pnpbios_interface_attach_device(struct pnp_bios_node *node) if (!proc_pnp) return -EIO; if (!pnpbios_dont_use_current_config) { - proc_create_data(name, 0644, proc_pnp, &pnpbios_proc_fops, + proc_create_data(name, 0644, proc_pnp, &pnpbios_proc_ops, (void *)(long)(node->handle)); } if (!proc_pnp_boot) return -EIO; - if (proc_create_data(name, 0644, proc_pnp_boot, &pnpbios_proc_fops, + if (proc_create_data(name, 0644, proc_pnp_boot, &pnpbios_proc_ops, (void *)(long)(node->handle + 0x100))) return 0; return -EIO; diff --git a/drivers/power/avs/Kconfig b/drivers/power/avs/Kconfig index b8fe166cd0d9..cdb4237bfd02 100644 --- a/drivers/power/avs/Kconfig +++ b/drivers/power/avs/Kconfig @@ -14,7 +14,7 @@ menuconfig POWER_AVS config QCOM_CPR tristate "QCOM Core Power Reduction (CPR) support" - depends on POWER_AVS + depends on POWER_AVS && HAS_IOMEM select PM_OPP select REGMAP help diff --git a/drivers/power/avs/qcom-cpr.c b/drivers/power/avs/qcom-cpr.c index 9192fb747653..bd7c3e48b386 100644 --- a/drivers/power/avs/qcom-cpr.c +++ b/drivers/power/avs/qcom-cpr.c @@ -517,7 +517,7 @@ static int cpr_scale(struct cpr_drv *drv, enum voltage_change_dir dir) dev_dbg(drv->dev, "UP: -> new_uV: %d last_uV: %d perf state: %u\n", new_uV, last_uV, cpr_get_cur_perf_state(drv)); - } else if (dir == DOWN) { + } else { if (desc->clamp_timer_interval && error_steps < desc->down_threshold) { /* @@ -567,7 +567,7 @@ static int cpr_scale(struct cpr_drv *drv, enum voltage_change_dir dir) /* Disable auto nack down */ reg_mask = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN; val = 0; - } else if (dir == DOWN) { + } else { /* Restore default threshold for UP */ reg_mask = RBCPR_CTL_UP_THRESHOLD_MASK; reg_mask <<= RBCPR_CTL_UP_THRESHOLD_SHIFT; @@ -1547,8 +1547,6 @@ static int cpr_pd_attach_dev(struct generic_pm_domain *domain, goto unlock; } - dev_dbg(drv->dev, "number of OPPs: %d\n", drv->num_corners); - drv->corners = devm_kcalloc(drv->dev, drv->num_corners, sizeof(*drv->corners), GFP_KERNEL); @@ -1586,6 +1584,9 @@ static int cpr_pd_attach_dev(struct generic_pm_domain *domain, acc_desc->enable_mask, acc_desc->enable_mask); + dev_info(drv->dev, "driver initialized with %u OPPs\n", + drv->num_corners); + unlock: mutex_unlock(&drv->lock); diff --git a/drivers/power/avs/rockchip-io-domain.c b/drivers/power/avs/rockchip-io-domain.c index 398fc954419e..eece97f97ef8 100644 --- a/drivers/power/avs/rockchip-io-domain.c +++ b/drivers/power/avs/rockchip-io-domain.c @@ -152,18 +152,18 @@ static void px30_iodomain_init(struct rockchip_iodomain *iod) int ret; u32 val; - /* if no VCCIO0 supply we should leave things alone */ + /* if no VCCIO6 supply we should leave things alone */ if (!iod->supplies[PX30_IO_VSEL_VCCIO6_SUPPLY_NUM].reg) return; /* - * set vccio0 iodomain to also use this framework + * set vccio6 iodomain to also use this framework * instead of a special gpio. */ val = PX30_IO_VSEL_VCCIO6_SRC | (PX30_IO_VSEL_VCCIO6_SRC << 16); ret = regmap_write(iod->grf, PX30_IO_VSEL, val); if (ret < 0) - dev_warn(iod->dev, "couldn't update vccio0 ctrl\n"); + dev_warn(iod->dev, "couldn't update vccio6 ctrl\n"); } static void rk3288_iodomain_init(struct rockchip_iodomain *iod) diff --git a/drivers/power/supply/cros_usbpd-charger.c b/drivers/power/supply/cros_usbpd-charger.c index ffad9ee03a68..30c3d37511c9 100644 --- a/drivers/power/supply/cros_usbpd-charger.c +++ b/drivers/power/supply/cros_usbpd-charger.c @@ -5,7 +5,6 @@ * Copyright (c) 2014 - 2018 Google, Inc */ -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig index bd21655c37a6..30190beeb6e9 100644 --- a/drivers/pwm/Kconfig +++ b/drivers/pwm/Kconfig @@ -100,7 +100,7 @@ config PWM_BCM_KONA config PWM_BCM2835 tristate "BCM2835 PWM support" - depends on ARCH_BCM2835 + depends on ARCH_BCM2835 || ARCH_BRCMSTB help PWM framework driver for BCM2835 controller (Raspberry Pi) @@ -328,7 +328,8 @@ config PWM_MXS config PWM_OMAP_DMTIMER tristate "OMAP Dual-Mode Timer PWM support" - depends on OF && ARCH_OMAP && OMAP_DM_TIMER + depends on OF + depends on OMAP_DM_TIMER || COMPILE_TEST help Generic PWM framework driver for OMAP Dual-Mode Timer PWM output @@ -490,7 +491,7 @@ config PWM_TEGRA To compile this driver as a module, choose M here: the module will be called pwm-tegra. -config PWM_TIECAP +config PWM_TIECAP tristate "ECAP PWM support" depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX || ARCH_KEYSTONE || ARCH_K3 help @@ -499,7 +500,7 @@ config PWM_TIECAP To compile this driver as a module, choose M here: the module will be called pwm-tiecap. -config PWM_TIEHRPWM +config PWM_TIEHRPWM tristate "EHRPWM PWM support" depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX || ARCH_K3 help diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index f877e77d9184..5a7f6598c05f 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -20,6 +20,9 @@ #include <dt-bindings/pwm/pwm.h> +#define CREATE_TRACE_POINTS +#include <trace/events/pwm.h> + #define MAX_PWMS 1024 static DEFINE_MUTEX(pwm_lookup_lock); @@ -114,6 +117,11 @@ static int pwm_device_request(struct pwm_device *pwm, const char *label) } } + if (pwm->chip->ops->get_state) { + pwm->chip->ops->get_state(pwm->chip, pwm, &pwm->state); + trace_pwm_get(pwm, &pwm->state); + } + set_bit(PWMF_REQUESTED, &pwm->flags); pwm->label = label; @@ -283,9 +291,6 @@ int pwmchip_add_with_polarity(struct pwm_chip *chip, pwm->hwpwm = i; pwm->state.polarity = polarity; - if (chip->ops->get_state) - chip->ops->get_state(chip, pwm, &pwm->state); - radix_tree_insert(&pwm_tree, pwm->pwm, pwm); } @@ -472,6 +477,8 @@ int pwm_apply_state(struct pwm_device *pwm, const struct pwm_state *state) if (err) return err; + trace_pwm_apply(pwm, state); + pwm->state = *state; } else { /* diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c index 9ba733467e26..6161e7e3e9ac 100644 --- a/drivers/pwm/pwm-atmel.c +++ b/drivers/pwm/pwm-atmel.c @@ -4,6 +4,19 @@ * * Copyright (C) 2013 Atmel Corporation * Bo Shen <voice.shen@atmel.com> + * + * Links to reference manuals for the supported PWM chips can be found in + * Documentation/arm/microchip.rst. + * + * Limitations: + * - Periods start with the inactive level. + * - Hardware has to be stopped in general to update settings. + * + * Software bugs/possible improvements: + * - When atmel_pwm_apply() is called with state->enabled=false a change in + * state->polarity isn't honored. + * - Instead of sleeping to wait for a completed period, the interrupt + * functionality could be used. */ #include <linux/clk.h> @@ -47,6 +60,8 @@ #define PWMV2_CPRD 0x0C #define PWMV2_CPRDUPD 0x10 +#define PWM_MAX_PRES 10 + struct atmel_pwm_registers { u8 period; u8 period_upd; @@ -55,8 +70,7 @@ struct atmel_pwm_registers { }; struct atmel_pwm_config { - u32 max_period; - u32 max_pres; + u32 period_bits; }; struct atmel_pwm_data { @@ -97,7 +111,7 @@ static inline u32 atmel_pwm_ch_readl(struct atmel_pwm_chip *chip, { unsigned long base = PWM_CH_REG_OFFSET + ch * PWM_CH_REG_SIZE; - return readl_relaxed(chip->base + base + offset); + return atmel_pwm_readl(chip, base + offset); } static inline void atmel_pwm_ch_writel(struct atmel_pwm_chip *chip, @@ -106,7 +120,7 @@ static inline void atmel_pwm_ch_writel(struct atmel_pwm_chip *chip, { unsigned long base = PWM_CH_REG_OFFSET + ch * PWM_CH_REG_SIZE; - writel_relaxed(val, chip->base + base + offset); + atmel_pwm_writel(chip, base + offset, val); } static int atmel_pwm_calculate_cprd_and_pres(struct pwm_chip *chip, @@ -115,17 +129,27 @@ static int atmel_pwm_calculate_cprd_and_pres(struct pwm_chip *chip, { struct atmel_pwm_chip *atmel_pwm = to_atmel_pwm_chip(chip); unsigned long long cycles = state->period; + int shift; /* Calculate the period cycles and prescale value */ cycles *= clk_get_rate(atmel_pwm->clk); do_div(cycles, NSEC_PER_SEC); - for (*pres = 0; cycles > atmel_pwm->data->cfg.max_period; cycles >>= 1) - (*pres)++; + /* + * The register for the period length is cfg.period_bits bits wide. + * So for each bit the number of clock cycles is wider divide the input + * clock frequency by two using pres and shift cprd accordingly. + */ + shift = fls(cycles) - atmel_pwm->data->cfg.period_bits; - if (*pres > atmel_pwm->data->cfg.max_pres) { + if (shift > PWM_MAX_PRES) { dev_err(chip->dev, "pres exceeds the maximum value\n"); return -EINVAL; + } else if (shift > 0) { + *pres = shift; + cycles >>= *pres; + } else { + *pres = 0; } *cprd = cycles; @@ -271,8 +295,48 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, return 0; } +static void atmel_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, + struct pwm_state *state) +{ + struct atmel_pwm_chip *atmel_pwm = to_atmel_pwm_chip(chip); + u32 sr, cmr; + + sr = atmel_pwm_readl(atmel_pwm, PWM_SR); + cmr = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, PWM_CMR); + + if (sr & (1 << pwm->hwpwm)) { + unsigned long rate = clk_get_rate(atmel_pwm->clk); + u32 cdty, cprd, pres; + u64 tmp; + + pres = cmr & PWM_CMR_CPRE_MSK; + + cprd = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, + atmel_pwm->data->regs.period); + tmp = (u64)cprd * NSEC_PER_SEC; + tmp <<= pres; + state->period = DIV64_U64_ROUND_UP(tmp, rate); + + cdty = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, + atmel_pwm->data->regs.duty); + tmp = (u64)cdty * NSEC_PER_SEC; + tmp <<= pres; + state->duty_cycle = DIV64_U64_ROUND_UP(tmp, rate); + + state->enabled = true; + } else { + state->enabled = false; + } + + if (cmr & PWM_CMR_CPOL) + state->polarity = PWM_POLARITY_INVERSED; + else + state->polarity = PWM_POLARITY_NORMAL; +} + static const struct pwm_ops atmel_pwm_ops = { .apply = atmel_pwm_apply, + .get_state = atmel_pwm_get_state, .owner = THIS_MODULE, }; @@ -285,8 +349,7 @@ static const struct atmel_pwm_data atmel_sam9rl_pwm_data = { }, .cfg = { /* 16 bits to keep period and duty. */ - .max_period = 0xffff, - .max_pres = 10, + .period_bits = 16, }, }; @@ -299,8 +362,7 @@ static const struct atmel_pwm_data atmel_sama5_pwm_data = { }, .cfg = { /* 16 bits to keep period and duty. */ - .max_period = 0xffff, - .max_pres = 10, + .period_bits = 16, }, }; @@ -313,8 +375,7 @@ static const struct atmel_pwm_data mchp_sam9x60_pwm_data = { }, .cfg = { /* 32 bits to keep period and duty. */ - .max_period = 0xffffffff, - .max_pres = 10, + .period_bits = 32, }, }; diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index 89497448d217..09c08dee099e 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -25,11 +25,39 @@ struct cros_ec_pwm_device { struct pwm_chip chip; }; +/** + * struct cros_ec_pwm - per-PWM driver data + * @duty_cycle: cached duty cycle + */ +struct cros_ec_pwm { + u16 duty_cycle; +}; + static inline struct cros_ec_pwm_device *pwm_to_cros_ec_pwm(struct pwm_chip *c) { return container_of(c, struct cros_ec_pwm_device, chip); } +static int cros_ec_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) +{ + struct cros_ec_pwm *channel; + + channel = kzalloc(sizeof(*channel), GFP_KERNEL); + if (!channel) + return -ENOMEM; + + pwm_set_chip_data(pwm, channel); + + return 0; +} + +static void cros_ec_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) +{ + struct cros_ec_pwm *channel = pwm_get_chip_data(pwm); + + kfree(channel); +} + static int cros_ec_pwm_set_duty(struct cros_ec_device *ec, u8 index, u16 duty) { struct { @@ -96,7 +124,9 @@ static int cros_ec_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct cros_ec_pwm_device *ec_pwm = pwm_to_cros_ec_pwm(chip); - int duty_cycle; + struct cros_ec_pwm *channel = pwm_get_chip_data(pwm); + u16 duty_cycle; + int ret; /* The EC won't let us change the period */ if (state->period != EC_PWM_MAX_DUTY) @@ -108,13 +138,20 @@ static int cros_ec_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, */ duty_cycle = state->enabled ? state->duty_cycle : 0; - return cros_ec_pwm_set_duty(ec_pwm->ec, pwm->hwpwm, duty_cycle); + ret = cros_ec_pwm_set_duty(ec_pwm->ec, pwm->hwpwm, duty_cycle); + if (ret < 0) + return ret; + + channel->duty_cycle = state->duty_cycle; + + return 0; } static void cros_ec_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { struct cros_ec_pwm_device *ec_pwm = pwm_to_cros_ec_pwm(chip); + struct cros_ec_pwm *channel = pwm_get_chip_data(pwm); int ret; ret = cros_ec_pwm_get_duty(ec_pwm->ec, pwm->hwpwm); @@ -126,8 +163,19 @@ static void cros_ec_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->enabled = (ret > 0); state->period = EC_PWM_MAX_DUTY; - /* Note that "disabled" and "duty cycle == 0" are treated the same */ - state->duty_cycle = ret; + /* + * Note that "disabled" and "duty cycle == 0" are treated the same. If + * the cached duty cycle is not zero, used the cached duty cycle. This + * ensures that the configured duty cycle is kept across a disable and + * enable operation and avoids potentially confusing consumers. + * + * For the case of the initial hardware readout, channel->duty_cycle + * will be 0 and the actual duty cycle read from the EC is used. + */ + if (ret == 0 && channel->duty_cycle > 0) + state->duty_cycle = channel->duty_cycle; + else + state->duty_cycle = ret; } static struct pwm_device * @@ -149,6 +197,8 @@ cros_ec_pwm_xlate(struct pwm_chip *pc, const struct of_phandle_args *args) } static const struct pwm_ops cros_ec_pwm_ops = { + .request = cros_ec_pwm_request, + .free = cros_ec_pwm_free, .get_state = cros_ec_pwm_get_state, .apply = cros_ec_pwm_apply, .owner = THIS_MODULE, diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c index ae11d8577f18..35a7ac42269c 100644 --- a/drivers/pwm/pwm-imx27.c +++ b/drivers/pwm/pwm-imx27.c @@ -85,6 +85,13 @@ struct pwm_imx27_chip { struct clk *clk_per; void __iomem *mmio_base; struct pwm_chip chip; + + /* + * The driver cannot read the current duty cycle from the hardware if + * the hardware is disabled. Cache the last programmed duty cycle + * value to return in that case. + */ + unsigned int duty_cycle; }; #define to_pwm_imx27_chip(chip) container_of(chip, struct pwm_imx27_chip, chip) @@ -155,14 +162,17 @@ static void pwm_imx27_get_state(struct pwm_chip *chip, tmp = NSEC_PER_SEC * (u64)(period + 2); state->period = DIV_ROUND_CLOSEST_ULL(tmp, pwm_clk); - /* PWMSAR can be read only if PWM is enabled */ - if (state->enabled) { + /* + * PWMSAR can be read only if PWM is enabled. If the PWM is disabled, + * use the cached value. + */ + if (state->enabled) val = readl(imx->mmio_base + MX3_PWMSAR); - tmp = NSEC_PER_SEC * (u64)(val); - state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, pwm_clk); - } else { - state->duty_cycle = 0; - } + else + val = imx->duty_cycle; + + tmp = NSEC_PER_SEC * (u64)(val); + state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, pwm_clk); if (!state->enabled) pwm_imx27_clk_disable_unprepare(chip); @@ -220,63 +230,68 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, pwm_get_state(pwm, &cstate); - if (state->enabled) { - c = clk_get_rate(imx->clk_per); - c *= state->period; - - do_div(c, 1000000000); - period_cycles = c; - - prescale = period_cycles / 0x10000 + 1; - - period_cycles /= prescale; - c = (unsigned long long)period_cycles * state->duty_cycle; - do_div(c, state->period); - duty_cycles = c; - - /* - * according to imx pwm RM, the real period value should be - * PERIOD value in PWMPR plus 2. - */ - if (period_cycles > 2) - period_cycles -= 2; - else - period_cycles = 0; - - /* - * Wait for a free FIFO slot if the PWM is already enabled, and - * flush the FIFO if the PWM was disabled and is about to be - * enabled. - */ - if (cstate.enabled) { - pwm_imx27_wait_fifo_slot(chip, pwm); - } else { - ret = pwm_imx27_clk_prepare_enable(chip); - if (ret) - return ret; - - pwm_imx27_sw_reset(chip); - } - - writel(duty_cycles, imx->mmio_base + MX3_PWMSAR); - writel(period_cycles, imx->mmio_base + MX3_PWMPR); - - cr = MX3_PWMCR_PRESCALER_SET(prescale) | - MX3_PWMCR_STOPEN | MX3_PWMCR_DOZEN | MX3_PWMCR_WAITEN | - FIELD_PREP(MX3_PWMCR_CLKSRC, MX3_PWMCR_CLKSRC_IPG_HIGH) | - MX3_PWMCR_DBGEN | MX3_PWMCR_EN; - - if (state->polarity == PWM_POLARITY_INVERSED) - cr |= FIELD_PREP(MX3_PWMCR_POUTC, - MX3_PWMCR_POUTC_INVERTED); - - writel(cr, imx->mmio_base + MX3_PWMCR); - } else if (cstate.enabled) { - writel(0, imx->mmio_base + MX3_PWMCR); + c = clk_get_rate(imx->clk_per); + c *= state->period; - pwm_imx27_clk_disable_unprepare(chip); + do_div(c, 1000000000); + period_cycles = c; + + prescale = period_cycles / 0x10000 + 1; + + period_cycles /= prescale; + c = (unsigned long long)period_cycles * state->duty_cycle; + do_div(c, state->period); + duty_cycles = c; + + /* + * according to imx pwm RM, the real period value should be PERIOD + * value in PWMPR plus 2. + */ + if (period_cycles > 2) + period_cycles -= 2; + else + period_cycles = 0; + + /* + * Wait for a free FIFO slot if the PWM is already enabled, and flush + * the FIFO if the PWM was disabled and is about to be enabled. + */ + if (cstate.enabled) { + pwm_imx27_wait_fifo_slot(chip, pwm); + } else { + ret = pwm_imx27_clk_prepare_enable(chip); + if (ret) + return ret; + + pwm_imx27_sw_reset(chip); } + writel(duty_cycles, imx->mmio_base + MX3_PWMSAR); + writel(period_cycles, imx->mmio_base + MX3_PWMPR); + + /* + * Store the duty cycle for future reference in cases where the + * MX3_PWMSAR register can't be read (i.e. when the PWM is disabled). + */ + imx->duty_cycle = duty_cycles; + + cr = MX3_PWMCR_PRESCALER_SET(prescale) | + MX3_PWMCR_STOPEN | MX3_PWMCR_DOZEN | MX3_PWMCR_WAITEN | + FIELD_PREP(MX3_PWMCR_CLKSRC, MX3_PWMCR_CLKSRC_IPG_HIGH) | + MX3_PWMCR_DBGEN; + + if (state->polarity == PWM_POLARITY_INVERSED) + cr |= FIELD_PREP(MX3_PWMCR_POUTC, + MX3_PWMCR_POUTC_INVERTED); + + if (state->enabled) + cr |= MX3_PWMCR_EN; + + writel(cr, imx->mmio_base + MX3_PWMCR); + + if (!state->enabled && cstate.enabled) + pwm_imx27_clk_disable_unprepare(chip); + return 0; } @@ -304,9 +319,13 @@ static int pwm_imx27_probe(struct platform_device *pdev) imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); if (IS_ERR(imx->clk_ipg)) { - dev_err(&pdev->dev, "getting ipg clock failed with %ld\n", - PTR_ERR(imx->clk_ipg)); - return PTR_ERR(imx->clk_ipg); + int ret = PTR_ERR(imx->clk_ipg); + + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, + "getting ipg clock failed with %d\n", + ret); + return ret; } imx->clk_per = devm_clk_get(&pdev->dev, "per"); diff --git a/drivers/pwm/pwm-mxs.c b/drivers/pwm/pwm-mxs.c index b14376b47ac8..f2e57fcf8f8b 100644 --- a/drivers/pwm/pwm-mxs.c +++ b/drivers/pwm/pwm-mxs.c @@ -25,12 +25,16 @@ #define PERIOD_PERIOD(p) ((p) & 0xffff) #define PERIOD_PERIOD_MAX 0x10000 #define PERIOD_ACTIVE_HIGH (3 << 16) +#define PERIOD_ACTIVE_LOW (2 << 16) +#define PERIOD_INACTIVE_HIGH (3 << 18) #define PERIOD_INACTIVE_LOW (2 << 18) +#define PERIOD_POLARITY_NORMAL (PERIOD_ACTIVE_HIGH | PERIOD_INACTIVE_LOW) +#define PERIOD_POLARITY_INVERSE (PERIOD_ACTIVE_LOW | PERIOD_INACTIVE_HIGH) #define PERIOD_CDIV(div) (((div) & 0x7) << 20) #define PERIOD_CDIV_MAX 8 -static const unsigned int cdiv[PERIOD_CDIV_MAX] = { - 1, 2, 4, 8, 16, 64, 256, 1024 +static const u8 cdiv_shift[PERIOD_CDIV_MAX] = { + 0, 1, 2, 3, 4, 6, 8, 10 }; struct mxs_pwm_chip { @@ -41,19 +45,34 @@ struct mxs_pwm_chip { #define to_mxs_pwm_chip(_chip) container_of(_chip, struct mxs_pwm_chip, chip) -static int mxs_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, - int duty_ns, int period_ns) +static int mxs_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, + const struct pwm_state *state) { struct mxs_pwm_chip *mxs = to_mxs_pwm_chip(chip); int ret, div = 0; unsigned int period_cycles, duty_cycles; unsigned long rate; unsigned long long c; + unsigned int pol_bits; + + /* + * If the PWM channel is disabled, make sure to turn on the + * clock before calling clk_get_rate() and writing to the + * registers. Otherwise, just keep it enabled. + */ + if (!pwm_is_enabled(pwm)) { + ret = clk_prepare_enable(mxs->clk); + if (ret) + return ret; + } + + if (!state->enabled && pwm_is_enabled(pwm)) + writel(1 << pwm->hwpwm, mxs->base + PWM_CTRL + CLR); rate = clk_get_rate(mxs->clk); while (1) { - c = rate / cdiv[div]; - c = c * period_ns; + c = rate >> cdiv_shift[div]; + c = c * state->period; do_div(c, 1000000000); if (c < PERIOD_PERIOD_MAX) break; @@ -63,62 +82,40 @@ static int mxs_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, } period_cycles = c; - c *= duty_ns; - do_div(c, period_ns); + c *= state->duty_cycle; + do_div(c, state->period); duty_cycles = c; /* - * If the PWM channel is disabled, make sure to turn on the clock - * before writing the register. Otherwise, keep it enabled. + * The data sheet the says registers must be written to in + * this order (ACTIVEn, then PERIODn). Also, the new settings + * only take effect at the beginning of a new period, avoiding + * glitches. */ - if (!pwm_is_enabled(pwm)) { - ret = clk_prepare_enable(mxs->clk); - if (ret) - return ret; - } + pol_bits = state->polarity == PWM_POLARITY_NORMAL ? + PERIOD_POLARITY_NORMAL : PERIOD_POLARITY_INVERSE; writel(duty_cycles << 16, - mxs->base + PWM_ACTIVE0 + pwm->hwpwm * 0x20); - writel(PERIOD_PERIOD(period_cycles) | PERIOD_ACTIVE_HIGH | - PERIOD_INACTIVE_LOW | PERIOD_CDIV(div), - mxs->base + PWM_PERIOD0 + pwm->hwpwm * 0x20); - - /* - * If the PWM is not enabled, turn the clock off again to save power. - */ - if (!pwm_is_enabled(pwm)) + mxs->base + PWM_ACTIVE0 + pwm->hwpwm * 0x20); + writel(PERIOD_PERIOD(period_cycles) | pol_bits | PERIOD_CDIV(div), + mxs->base + PWM_PERIOD0 + pwm->hwpwm * 0x20); + + if (state->enabled) { + if (!pwm_is_enabled(pwm)) { + /* + * The clock was enabled above. Just enable + * the channel in the control register. + */ + writel(1 << pwm->hwpwm, mxs->base + PWM_CTRL + SET); + } + } else { clk_disable_unprepare(mxs->clk); - - return 0; -} - -static int mxs_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) -{ - struct mxs_pwm_chip *mxs = to_mxs_pwm_chip(chip); - int ret; - - ret = clk_prepare_enable(mxs->clk); - if (ret) - return ret; - - writel(1 << pwm->hwpwm, mxs->base + PWM_CTRL + SET); - + } return 0; } -static void mxs_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) -{ - struct mxs_pwm_chip *mxs = to_mxs_pwm_chip(chip); - - writel(1 << pwm->hwpwm, mxs->base + PWM_CTRL + CLR); - - clk_disable_unprepare(mxs->clk); -} - static const struct pwm_ops mxs_pwm_ops = { - .config = mxs_pwm_config, - .enable = mxs_pwm_enable, - .disable = mxs_pwm_disable, + .apply = mxs_pwm_apply, .owner = THIS_MODULE, }; @@ -142,6 +139,8 @@ static int mxs_pwm_probe(struct platform_device *pdev) mxs->chip.dev = &pdev->dev; mxs->chip.ops = &mxs_pwm_ops; + mxs->chip.of_xlate = of_pwm_xlate_with_flags; + mxs->chip.of_pwm_n_cells = 3; mxs->chip.base = -1; ret = of_property_read_u32(np, "fsl,pwm-number", &mxs->chip.npwm); diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c index 00772fc53490..88a3c5690fea 100644 --- a/drivers/pwm/pwm-omap-dmtimer.c +++ b/drivers/pwm/pwm-omap-dmtimer.c @@ -256,7 +256,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) if (!timer_pdev) { dev_err(&pdev->dev, "Unable to find Timer pdev\n"); ret = -ENODEV; - goto put; + goto err_find_timer_pdev; } timer_pdata = dev_get_platdata(&timer_pdev->dev); @@ -264,7 +264,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) dev_dbg(&pdev->dev, "dmtimer pdata structure NULL, deferring probe\n"); ret = -EPROBE_DEFER; - goto put; + goto err_platdata; } pdata = timer_pdata->timer_ops; @@ -283,30 +283,25 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) !pdata->write_counter) { dev_err(&pdev->dev, "Incomplete dmtimer pdata structure\n"); ret = -EINVAL; - goto put; + goto err_platdata; } if (!of_get_property(timer, "ti,timer-pwm", NULL)) { dev_err(&pdev->dev, "Missing ti,timer-pwm capability\n"); ret = -ENODEV; - goto put; + goto err_timer_property; } dm_timer = pdata->request_by_node(timer); if (!dm_timer) { ret = -EPROBE_DEFER; - goto put; + goto err_request_timer; } -put: - of_node_put(timer); - if (ret < 0) - return ret; - omap = devm_kzalloc(&pdev->dev, sizeof(*omap), GFP_KERNEL); if (!omap) { - pdata->free(dm_timer); - return -ENOMEM; + ret = -ENOMEM; + goto err_alloc_omap; } omap->pdata = pdata; @@ -339,27 +334,56 @@ put: ret = pwmchip_add(&omap->chip); if (ret < 0) { dev_err(&pdev->dev, "failed to register PWM\n"); - omap->pdata->free(omap->dm_timer); - return ret; + goto err_pwmchip_add; } + of_node_put(timer); + platform_set_drvdata(pdev, omap); return 0; + +err_pwmchip_add: + + /* + * *omap is allocated using devm_kzalloc, + * so no free necessary here + */ +err_alloc_omap: + + pdata->free(dm_timer); +err_request_timer: + +err_timer_property: +err_platdata: + + put_device(&timer_pdev->dev); +err_find_timer_pdev: + + of_node_put(timer); + + return ret; } static int pwm_omap_dmtimer_remove(struct platform_device *pdev) { struct pwm_omap_dmtimer_chip *omap = platform_get_drvdata(pdev); + int ret; + + ret = pwmchip_remove(&omap->chip); + if (ret) + return ret; if (pm_runtime_active(&omap->dm_timer_pdev->dev)) omap->pdata->stop(omap->dm_timer); omap->pdata->free(omap->dm_timer); + put_device(&omap->dm_timer_pdev->dev); + mutex_destroy(&omap->mutex); - return pwmchip_remove(&omap->chip); + return 0; } static const struct of_device_id pwm_omap_dmtimer_of_match[] = { diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c index 168684b02ebc..b07bdca3d510 100644 --- a/drivers/pwm/pwm-pca9685.c +++ b/drivers/pwm/pwm-pca9685.c @@ -159,13 +159,9 @@ static void pca9685_pwm_gpio_set(struct gpio_chip *gpio, unsigned int offset, static void pca9685_pwm_gpio_free(struct gpio_chip *gpio, unsigned int offset) { struct pca9685 *pca = gpiochip_get_data(gpio); - struct pwm_device *pwm; pca9685_pwm_gpio_set(gpio, offset, 0); pm_runtime_put(pca->chip.dev); - mutex_lock(&pca->lock); - pwm = &pca->chip.pwms[offset]; - mutex_unlock(&pca->lock); } static int pca9685_pwm_gpio_get_direction(struct gpio_chip *chip, diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c index 852eb2347954..2685577b6dd4 100644 --- a/drivers/pwm/pwm-rcar.c +++ b/drivers/pwm/pwm-rcar.c @@ -3,6 +3,9 @@ * R-Car PWM Timer driver * * Copyright (C) 2015 Renesas Electronics Corporation + * + * Limitations: + * - The hardware cannot generate a 0% duty cycle. */ #include <linux/clk.h> @@ -161,11 +164,9 @@ static int rcar_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct rcar_pwm_chip *rp = to_rcar_pwm_chip(chip); - struct pwm_state cur_state; int div, ret; /* This HW/driver only supports normal polarity */ - pwm_get_state(pwm, &cur_state); if (state->polarity != PWM_POLARITY_NORMAL) return -ENOTSUPP; diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index 7ff48c14fae8..d3be944f2ae9 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -377,9 +377,7 @@ static int stm32_pwm_config(struct stm32_pwm *priv, int ch, else regmap_update_bits(priv->regmap, TIM_CCMR2, mask, ccmr); - regmap_update_bits(priv->regmap, TIM_BDTR, - TIM_BDTR_MOE | TIM_BDTR_AOE, - TIM_BDTR_MOE | TIM_BDTR_AOE); + regmap_update_bits(priv->regmap, TIM_BDTR, TIM_BDTR_MOE, TIM_BDTR_MOE); return 0; } diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index 581d23287333..3e3efa6c768f 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -3,6 +3,10 @@ * Driver for Allwinner sun4i Pulse Width Modulation Controller * * Copyright (C) 2014 Alexandre Belloni <alexandre.belloni@free-electrons.com> + * + * Limitations: + * - When outputing the source clock directly, the PWM logic will be bypassed + * and the currently running period is not guaranteed to be completed */ #include <linux/bitops.h> @@ -16,6 +20,7 @@ #include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/pwm.h> +#include <linux/reset.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/time.h> @@ -72,12 +77,15 @@ static const u32 prescaler_table[] = { struct sun4i_pwm_data { bool has_prescaler_bypass; + bool has_direct_mod_clk_output; unsigned int npwm; }; struct sun4i_pwm_chip { struct pwm_chip chip; + struct clk *bus_clk; struct clk *clk; + struct reset_control *rst; void __iomem *base; spinlock_t ctrl_lock; const struct sun4i_pwm_data *data; @@ -115,6 +123,20 @@ static void sun4i_pwm_get_state(struct pwm_chip *chip, val = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); + /* + * PWM chapter in H6 manual has a diagram which explains that if bypass + * bit is set, no other setting has any meaning. Even more, experiment + * proved that also enable bit is ignored in this case. + */ + if ((val & BIT_CH(PWM_BYPASS, pwm->hwpwm)) && + sun4i_pwm->data->has_direct_mod_clk_output) { + state->period = DIV_ROUND_UP_ULL(NSEC_PER_SEC, clk_rate); + state->duty_cycle = DIV_ROUND_UP_ULL(state->period, 2); + state->polarity = PWM_POLARITY_NORMAL; + state->enabled = true; + return; + } + if ((PWM_REG_PRESCAL(val, pwm->hwpwm) == PWM_PRESCAL_MASK) && sun4i_pwm->data->has_prescaler_bypass) prescaler = 1; @@ -146,13 +168,24 @@ static void sun4i_pwm_get_state(struct pwm_chip *chip, static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, const struct pwm_state *state, - u32 *dty, u32 *prd, unsigned int *prsclr) + u32 *dty, u32 *prd, unsigned int *prsclr, + bool *bypass) { u64 clk_rate, div = 0; - unsigned int pval, prescaler = 0; + unsigned int prescaler = 0; clk_rate = clk_get_rate(sun4i_pwm->clk); + *bypass = sun4i_pwm->data->has_direct_mod_clk_output && + state->enabled && + (state->period * clk_rate >= NSEC_PER_SEC) && + (state->period * clk_rate < 2 * NSEC_PER_SEC) && + (state->duty_cycle * clk_rate * 2 >= NSEC_PER_SEC); + + /* Skip calculation of other parameters if we bypass them */ + if (*bypass) + return 0; + if (sun4i_pwm->data->has_prescaler_bypass) { /* First, test without any prescaler when available */ prescaler = PWM_PRESCAL_MASK; @@ -170,9 +203,11 @@ static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, if (prescaler == 0) { /* Go up from the first divider */ for (prescaler = 0; prescaler < PWM_PRESCAL_MASK; prescaler++) { - if (!prescaler_table[prescaler]) + unsigned int pval = prescaler_table[prescaler]; + + if (!pval) continue; - pval = prescaler_table[prescaler]; + div = clk_rate; do_div(div, pval); div = div * state->period; @@ -199,10 +234,11 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, { struct sun4i_pwm_chip *sun4i_pwm = to_sun4i_pwm_chip(chip); struct pwm_state cstate; - u32 ctrl; + u32 ctrl, duty = 0, period = 0, val; int ret; - unsigned int delay_us; + unsigned int delay_us, prescaler = 0; unsigned long now; + bool bypass; pwm_get_state(pwm, &cstate); @@ -214,46 +250,52 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, } } + ret = sun4i_pwm_calculate(sun4i_pwm, state, &duty, &period, &prescaler, + &bypass); + if (ret) { + dev_err(chip->dev, "period exceeds the maximum value\n"); + if (!cstate.enabled) + clk_disable_unprepare(sun4i_pwm->clk); + return ret; + } + spin_lock(&sun4i_pwm->ctrl_lock); ctrl = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); - if ((cstate.period != state->period) || - (cstate.duty_cycle != state->duty_cycle)) { - u32 period, duty, val; - unsigned int prescaler; - - ret = sun4i_pwm_calculate(sun4i_pwm, state, - &duty, &period, &prescaler); - if (ret) { - dev_err(chip->dev, "period exceeds the maximum value\n"); + if (sun4i_pwm->data->has_direct_mod_clk_output) { + if (bypass) { + ctrl |= BIT_CH(PWM_BYPASS, pwm->hwpwm); + /* We can skip other parameter */ + sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); spin_unlock(&sun4i_pwm->ctrl_lock); - if (!cstate.enabled) - clk_disable_unprepare(sun4i_pwm->clk); - return ret; + return 0; } - if (PWM_REG_PRESCAL(ctrl, pwm->hwpwm) != prescaler) { - /* Prescaler changed, the clock has to be gated */ - ctrl &= ~BIT_CH(PWM_CLK_GATING, pwm->hwpwm); - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); + ctrl &= ~BIT_CH(PWM_BYPASS, pwm->hwpwm); + } - ctrl &= ~BIT_CH(PWM_PRESCAL_MASK, pwm->hwpwm); - ctrl |= BIT_CH(prescaler, pwm->hwpwm); - } + if (PWM_REG_PRESCAL(ctrl, pwm->hwpwm) != prescaler) { + /* Prescaler changed, the clock has to be gated */ + ctrl &= ~BIT_CH(PWM_CLK_GATING, pwm->hwpwm); + sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); - val = (duty & PWM_DTY_MASK) | PWM_PRD(period); - sun4i_pwm_writel(sun4i_pwm, val, PWM_CH_PRD(pwm->hwpwm)); - sun4i_pwm->next_period[pwm->hwpwm] = jiffies + - usecs_to_jiffies(cstate.period / 1000 + 1); - sun4i_pwm->needs_delay[pwm->hwpwm] = true; + ctrl &= ~BIT_CH(PWM_PRESCAL_MASK, pwm->hwpwm); + ctrl |= BIT_CH(prescaler, pwm->hwpwm); } + val = (duty & PWM_DTY_MASK) | PWM_PRD(period); + sun4i_pwm_writel(sun4i_pwm, val, PWM_CH_PRD(pwm->hwpwm)); + sun4i_pwm->next_period[pwm->hwpwm] = jiffies + + usecs_to_jiffies(cstate.period / 1000 + 1); + sun4i_pwm->needs_delay[pwm->hwpwm] = true; + if (state->polarity != PWM_POLARITY_NORMAL) ctrl &= ~BIT_CH(PWM_ACT_STATE, pwm->hwpwm); else ctrl |= BIT_CH(PWM_ACT_STATE, pwm->hwpwm); ctrl |= BIT_CH(PWM_CLK_GATING, pwm->hwpwm); + if (state->enabled) { ctrl |= BIT_CH(PWM_EN, pwm->hwpwm); } else if (!sun4i_pwm->needs_delay[pwm->hwpwm]) { @@ -319,6 +361,12 @@ static const struct sun4i_pwm_data sun4i_pwm_single_bypass = { .npwm = 1, }; +static const struct sun4i_pwm_data sun50i_h6_pwm_data = { + .has_prescaler_bypass = true, + .has_direct_mod_clk_output = true, + .npwm = 2, +}; + static const struct of_device_id sun4i_pwm_dt_ids[] = { { .compatible = "allwinner,sun4i-a10-pwm", @@ -336,6 +384,9 @@ static const struct of_device_id sun4i_pwm_dt_ids[] = { .compatible = "allwinner,sun8i-h3-pwm", .data = &sun4i_pwm_single_bypass, }, { + .compatible = "allwinner,sun50i-h6-pwm", + .data = &sun50i_h6_pwm_data, + }, { /* sentinel */ }, }; @@ -360,9 +411,69 @@ static int sun4i_pwm_probe(struct platform_device *pdev) if (IS_ERR(pwm->base)) return PTR_ERR(pwm->base); - pwm->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pwm->clk)) + /* + * All hardware variants need a source clock that is divided and + * then feeds the counter that defines the output wave form. In the + * device tree this clock is either unnamed or called "mod". + * Some variants (e.g. H6) need another clock to access the + * hardware registers; this is called "bus". + * So we request "mod" first (and ignore the corner case that a + * parent provides a "mod" clock while the right one would be the + * unnamed one of the PWM device) and if this is not found we fall + * back to the first clock of the PWM. + */ + pwm->clk = devm_clk_get_optional(&pdev->dev, "mod"); + if (IS_ERR(pwm->clk)) { + if (PTR_ERR(pwm->clk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "get mod clock failed %pe\n", + pwm->clk); return PTR_ERR(pwm->clk); + } + + if (!pwm->clk) { + pwm->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(pwm->clk)) { + if (PTR_ERR(pwm->clk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "get unnamed clock failed %pe\n", + pwm->clk); + return PTR_ERR(pwm->clk); + } + } + + pwm->bus_clk = devm_clk_get_optional(&pdev->dev, "bus"); + if (IS_ERR(pwm->bus_clk)) { + if (PTR_ERR(pwm->bus_clk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "get bus clock failed %pe\n", + pwm->bus_clk); + return PTR_ERR(pwm->bus_clk); + } + + pwm->rst = devm_reset_control_get_optional_shared(&pdev->dev, NULL); + if (IS_ERR(pwm->rst)) { + if (PTR_ERR(pwm->rst) != -EPROBE_DEFER) + dev_err(&pdev->dev, "get reset failed %pe\n", + pwm->rst); + return PTR_ERR(pwm->rst); + } + + /* Deassert reset */ + ret = reset_control_deassert(pwm->rst); + if (ret) { + dev_err(&pdev->dev, "cannot deassert reset control: %pe\n", + ERR_PTR(ret)); + return ret; + } + + /* + * We're keeping the bus clock on for the sake of simplicity. + * Actually it only needs to be on for hardware register accesses. + */ + ret = clk_prepare_enable(pwm->bus_clk); + if (ret) { + dev_err(&pdev->dev, "cannot prepare and enable bus_clk %pe\n", + ERR_PTR(ret)); + goto err_bus; + } pwm->chip.dev = &pdev->dev; pwm->chip.ops = &sun4i_pwm_ops; @@ -376,19 +487,34 @@ static int sun4i_pwm_probe(struct platform_device *pdev) ret = pwmchip_add(&pwm->chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); - return ret; + goto err_pwm_add; } platform_set_drvdata(pdev, pwm); return 0; + +err_pwm_add: + clk_disable_unprepare(pwm->bus_clk); +err_bus: + reset_control_assert(pwm->rst); + + return ret; } static int sun4i_pwm_remove(struct platform_device *pdev) { struct sun4i_pwm_chip *pwm = platform_get_drvdata(pdev); + int ret; + + ret = pwmchip_remove(&pwm->chip); + if (ret) + return ret; + + clk_disable_unprepare(pwm->bus_clk); + reset_control_assert(pwm->rst); - return pwmchip_remove(&pwm->chip); + return 0; } static struct platform_driver sun4i_pwm_driver = { diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig index 97bfdd47954f..074a2ef55943 100644 --- a/drivers/regulator/Kconfig +++ b/drivers/regulator/Kconfig @@ -209,6 +209,7 @@ config REGULATOR_BD71828 config REGULATOR_BD718XX tristate "ROHM BD71837 Power Regulator" depends on MFD_ROHM_BD718XX + select REGULATOR_ROHM help This driver supports voltage regulators on ROHM BD71837 PMIC. This will enable support for the software controllable buck @@ -823,6 +824,9 @@ config REGULATOR_RN5T618 Say y here to support the regulators found on Ricoh RN5T567, RN5T618 or RC5T619 PMIC. +config REGULATOR_ROHM + tristate + config REGULATOR_RT5033 tristate "Richtek RT5033 Regulators" depends on MFD_RT5033 diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile index 07bc977c52b0..c0d6b96ebd78 100644 --- a/drivers/regulator/Makefile +++ b/drivers/regulator/Makefile @@ -102,6 +102,7 @@ obj-$(CONFIG_REGULATOR_PCF50633) += pcf50633-regulator.o obj-$(CONFIG_REGULATOR_RC5T583) += rc5t583-regulator.o obj-$(CONFIG_REGULATOR_RK808) += rk808-regulator.o obj-$(CONFIG_REGULATOR_RN5T618) += rn5t618-regulator.o +obj-$(CONFIG_REGULATOR_ROHM) += rohm-regulator.o obj-$(CONFIG_REGULATOR_RT5033) += rt5033-regulator.o obj-$(CONFIG_REGULATOR_S2MPA01) += s2mpa01.o obj-$(CONFIG_REGULATOR_S2MPS11) += s2mps11.o diff --git a/drivers/regulator/bd718x7-regulator.c b/drivers/regulator/bd718x7-regulator.c index 8f9b2d8eaf10..cf3872837abc 100644 --- a/drivers/regulator/bd718x7-regulator.c +++ b/drivers/regulator/bd718x7-regulator.c @@ -318,6 +318,7 @@ struct reg_init { }; struct bd718xx_regulator_data { struct regulator_desc desc; + const struct rohm_dvs_config dvs; const struct reg_init init; const struct reg_init *additional_inits; int additional_init_amnt; @@ -349,133 +350,15 @@ static const struct reg_init bd71837_ldo6_inits[] = { }, }; -#define NUM_DVS_BUCKS 4 - -struct of_dvs_setting { - const char *prop; - unsigned int reg; -}; - -static int set_dvs_levels(const struct of_dvs_setting *dvs, - struct device_node *np, - const struct regulator_desc *desc, - struct regmap *regmap) -{ - int ret, i; - unsigned int uv; - - ret = of_property_read_u32(np, dvs->prop, &uv); - if (ret) { - if (ret != -EINVAL) - return ret; - return 0; - } - - for (i = 0; i < desc->n_voltages; i++) { - ret = regulator_desc_list_voltage_linear_range(desc, i); - if (ret < 0) - continue; - if (ret == uv) { - i <<= ffs(desc->vsel_mask) - 1; - ret = regmap_update_bits(regmap, dvs->reg, - DVS_BUCK_RUN_MASK, i); - break; - } - } - return ret; -} - -static int buck4_set_hw_dvs_levels(struct device_node *np, +static int buck_set_hw_dvs_levels(struct device_node *np, const struct regulator_desc *desc, struct regulator_config *cfg) { - int ret, i; - const struct of_dvs_setting dvs[] = { - { - .prop = "rohm,dvs-run-voltage", - .reg = BD71837_REG_BUCK4_VOLT_RUN, - }, - }; + struct bd718xx_regulator_data *data; - for (i = 0; i < ARRAY_SIZE(dvs); i++) { - ret = set_dvs_levels(&dvs[i], np, desc, cfg->regmap); - if (ret) - break; - } - return ret; -} -static int buck3_set_hw_dvs_levels(struct device_node *np, - const struct regulator_desc *desc, - struct regulator_config *cfg) -{ - int ret, i; - const struct of_dvs_setting dvs[] = { - { - .prop = "rohm,dvs-run-voltage", - .reg = BD71837_REG_BUCK3_VOLT_RUN, - }, - }; + data = container_of(desc, struct bd718xx_regulator_data, desc); - for (i = 0; i < ARRAY_SIZE(dvs); i++) { - ret = set_dvs_levels(&dvs[i], np, desc, cfg->regmap); - if (ret) - break; - } - return ret; -} - -static int buck2_set_hw_dvs_levels(struct device_node *np, - const struct regulator_desc *desc, - struct regulator_config *cfg) -{ - int ret, i; - const struct of_dvs_setting dvs[] = { - { - .prop = "rohm,dvs-run-voltage", - .reg = BD718XX_REG_BUCK2_VOLT_RUN, - }, - { - .prop = "rohm,dvs-idle-voltage", - .reg = BD718XX_REG_BUCK2_VOLT_IDLE, - }, - }; - - - - for (i = 0; i < ARRAY_SIZE(dvs); i++) { - ret = set_dvs_levels(&dvs[i], np, desc, cfg->regmap); - if (ret) - break; - } - return ret; -} - -static int buck1_set_hw_dvs_levels(struct device_node *np, - const struct regulator_desc *desc, - struct regulator_config *cfg) -{ - int ret, i; - const struct of_dvs_setting dvs[] = { - { - .prop = "rohm,dvs-run-voltage", - .reg = BD718XX_REG_BUCK1_VOLT_RUN, - }, - { - .prop = "rohm,dvs-idle-voltage", - .reg = BD718XX_REG_BUCK1_VOLT_IDLE, - }, - { - .prop = "rohm,dvs-suspend-voltage", - .reg = BD718XX_REG_BUCK1_VOLT_SUSP, - }, - }; - - for (i = 0; i < ARRAY_SIZE(dvs); i++) { - ret = set_dvs_levels(&dvs[i], np, desc, cfg->regmap); - if (ret) - break; - } - return ret; + return rohm_regulator_set_dvs_levels(&data->dvs, np, desc, cfg->regmap); } static const struct bd718xx_regulator_data bd71847_regulators[] = { @@ -496,7 +379,17 @@ static const struct bd718xx_regulator_data bd71847_regulators[] = { .enable_reg = BD718XX_REG_BUCK1_CTRL, .enable_mask = BD718XX_BUCK_EN, .owner = THIS_MODULE, - .of_parse_cb = buck1_set_hw_dvs_levels, + .of_parse_cb = buck_set_hw_dvs_levels, + }, + .dvs = { + .level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE | + ROHM_DVS_LEVEL_SUSPEND, + .run_reg = BD718XX_REG_BUCK1_VOLT_RUN, + .run_mask = DVS_BUCK_RUN_MASK, + .idle_reg = BD718XX_REG_BUCK1_VOLT_IDLE, + .idle_mask = DVS_BUCK_RUN_MASK, + .suspend_reg = BD718XX_REG_BUCK1_VOLT_SUSP, + .suspend_mask = DVS_BUCK_RUN_MASK, }, .init = { .reg = BD718XX_REG_BUCK1_CTRL, @@ -520,7 +413,14 @@ static const struct bd718xx_regulator_data bd71847_regulators[] = { .enable_reg = BD718XX_REG_BUCK2_CTRL, .enable_mask = BD718XX_BUCK_EN, .owner = THIS_MODULE, - .of_parse_cb = buck2_set_hw_dvs_levels, + .of_parse_cb = buck_set_hw_dvs_levels, + }, + .dvs = { + .level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE, + .run_reg = BD718XX_REG_BUCK2_VOLT_RUN, + .run_mask = DVS_BUCK_RUN_MASK, + .idle_reg = BD718XX_REG_BUCK2_VOLT_IDLE, + .idle_mask = DVS_BUCK_RUN_MASK, }, .init = { .reg = BD718XX_REG_BUCK2_CTRL, @@ -792,7 +692,17 @@ static const struct bd718xx_regulator_data bd71837_regulators[] = { .enable_reg = BD718XX_REG_BUCK1_CTRL, .enable_mask = BD718XX_BUCK_EN, .owner = THIS_MODULE, - .of_parse_cb = buck1_set_hw_dvs_levels, + .of_parse_cb = buck_set_hw_dvs_levels, + }, + .dvs = { + .level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE | + ROHM_DVS_LEVEL_SUSPEND, + .run_reg = BD718XX_REG_BUCK1_VOLT_RUN, + .run_mask = DVS_BUCK_RUN_MASK, + .idle_reg = BD718XX_REG_BUCK1_VOLT_IDLE, + .idle_mask = DVS_BUCK_RUN_MASK, + .suspend_reg = BD718XX_REG_BUCK1_VOLT_SUSP, + .suspend_mask = DVS_BUCK_RUN_MASK, }, .init = { .reg = BD718XX_REG_BUCK1_CTRL, @@ -816,7 +726,14 @@ static const struct bd718xx_regulator_data bd71837_regulators[] = { .enable_reg = BD718XX_REG_BUCK2_CTRL, .enable_mask = BD718XX_BUCK_EN, .owner = THIS_MODULE, - .of_parse_cb = buck2_set_hw_dvs_levels, + .of_parse_cb = buck_set_hw_dvs_levels, + }, + .dvs = { + .level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE, + .run_reg = BD718XX_REG_BUCK2_VOLT_RUN, + .run_mask = DVS_BUCK_RUN_MASK, + .idle_reg = BD718XX_REG_BUCK2_VOLT_IDLE, + .idle_mask = DVS_BUCK_RUN_MASK, }, .init = { .reg = BD718XX_REG_BUCK2_CTRL, @@ -840,7 +757,12 @@ static const struct bd718xx_regulator_data bd71837_regulators[] = { .enable_reg = BD71837_REG_BUCK3_CTRL, .enable_mask = BD718XX_BUCK_EN, .owner = THIS_MODULE, - .of_parse_cb = buck3_set_hw_dvs_levels, + .of_parse_cb = buck_set_hw_dvs_levels, + }, + .dvs = { + .level_map = ROHM_DVS_LEVEL_RUN, + .run_reg = BD71837_REG_BUCK3_VOLT_RUN, + .run_mask = DVS_BUCK_RUN_MASK, }, .init = { .reg = BD71837_REG_BUCK3_CTRL, @@ -864,7 +786,12 @@ static const struct bd718xx_regulator_data bd71837_regulators[] = { .enable_reg = BD71837_REG_BUCK4_CTRL, .enable_mask = BD718XX_BUCK_EN, .owner = THIS_MODULE, - .of_parse_cb = buck4_set_hw_dvs_levels, + .of_parse_cb = buck_set_hw_dvs_levels, + }, + .dvs = { + .level_map = ROHM_DVS_LEVEL_RUN, + .run_reg = BD71837_REG_BUCK4_VOLT_RUN, + .run_mask = DVS_BUCK_RUN_MASK, }, .init = { .reg = BD71837_REG_BUCK4_CTRL, @@ -1150,6 +1077,7 @@ static int bd718xx_probe(struct platform_device *pdev) bool use_snvs; const struct bd718xx_regulator_data *reg_data; unsigned int num_reg_data; + enum rohm_chip_type chip = platform_get_device_id(pdev)->driver_data; mfd = dev_get_drvdata(pdev->dev.parent); if (!mfd) { @@ -1158,7 +1086,7 @@ static int bd718xx_probe(struct platform_device *pdev) goto err; } - switch (mfd->chip.chip_type) { + switch (chip) { case ROHM_CHIP_TYPE_BD71837: reg_data = bd71837_regulators; num_reg_data = ARRAY_SIZE(bd71837_regulators); @@ -1275,11 +1203,19 @@ err: return err; } +static const struct platform_device_id bd718x7_pmic_id[] = { + { "bd71837-pmic", ROHM_CHIP_TYPE_BD71837 }, + { "bd71847-pmic", ROHM_CHIP_TYPE_BD71847 }, + { }, +}; +MODULE_DEVICE_TABLE(platform, bd718x7_pmic_id); + static struct platform_driver bd718xx_regulator = { .driver = { .name = "bd718xx-pmic", }, .probe = bd718xx_probe, + .id_table = bd718x7_pmic_id, }; module_platform_driver(bd718xx_regulator); diff --git a/drivers/regulator/rohm-regulator.c b/drivers/regulator/rohm-regulator.c new file mode 100644 index 000000000000..399002383b28 --- /dev/null +++ b/drivers/regulator/rohm-regulator.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2020 ROHM Semiconductors + +#include <linux/errno.h> +#include <linux/mfd/rohm-generic.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/regmap.h> +#include <linux/regulator/driver.h> + +static int set_dvs_level(const struct regulator_desc *desc, + struct device_node *np, struct regmap *regmap, + char *prop, unsigned int reg, unsigned int mask, + unsigned int omask, unsigned int oreg) +{ + int ret, i; + uint32_t uv; + + ret = of_property_read_u32(np, prop, &uv); + if (ret) { + if (ret != -EINVAL) + return ret; + return 0; + } + + if (uv == 0) { + if (omask) + return regmap_update_bits(regmap, oreg, omask, 0); + } + for (i = 0; i < desc->n_voltages; i++) { + ret = regulator_desc_list_voltage_linear_range(desc, i); + if (ret < 0) + continue; + if (ret == uv) { + i <<= ffs(desc->vsel_mask) - 1; + ret = regmap_update_bits(regmap, reg, mask, i); + if (omask && !ret) + ret = regmap_update_bits(regmap, oreg, omask, + omask); + break; + } + } + return ret; +} + +int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dvs, + struct device_node *np, + const struct regulator_desc *desc, + struct regmap *regmap) +{ + int i, ret = 0; + char *prop; + unsigned int reg, mask, omask, oreg = desc->enable_reg; + + for (i = 0; i < ROHM_DVS_LEVEL_MAX && !ret; i++) { + if (dvs->level_map & (1 << i)) { + switch (i + 1) { + case ROHM_DVS_LEVEL_RUN: + prop = "rohm,dvs-run-voltage"; + reg = dvs->run_reg; + mask = dvs->run_mask; + omask = dvs->run_on_mask; + break; + case ROHM_DVS_LEVEL_IDLE: + prop = "rohm,dvs-idle-voltage"; + reg = dvs->idle_reg; + mask = dvs->idle_mask; + omask = dvs->idle_on_mask; + break; + case ROHM_DVS_LEVEL_SUSPEND: + prop = "rohm,dvs-suspend-voltage"; + reg = dvs->suspend_reg; + mask = dvs->suspend_mask; + omask = dvs->suspend_on_mask; + break; + case ROHM_DVS_LEVEL_LPSR: + prop = "rohm,dvs-lpsr-voltage"; + reg = dvs->lpsr_reg; + mask = dvs->lpsr_mask; + omask = dvs->lpsr_on_mask; + break; + default: + return -EINVAL; + } + ret = set_dvs_level(desc, np, regmap, prop, reg, mask, + omask, oreg); + } + } + return ret; +} +EXPORT_SYMBOL(rohm_regulator_set_dvs_levels); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>"); +MODULE_DESCRIPTION("Generic helpers for ROHM PMIC regulator drivers"); diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index 94afdde4bc9f..de3862c15fcc 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -23,6 +23,16 @@ config IMX_REMOTEPROC It's safe to say N here. +config MTK_SCP + tristate "Mediatek SCP support" + depends on ARCH_MEDIATEK + select RPMSG_MTK_SCP + help + Say y here to support Mediatek's System Companion Processor (SCP) via + the remote processor framework. + + It's safe to say N here. + config OMAP_REMOTEPROC tristate "OMAP remoteproc support" depends on ARCH_OMAP4 || SOC_OMAP5 diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile index 00f09e658cb3..e30a1b15fbac 100644 --- a/drivers/remoteproc/Makefile +++ b/drivers/remoteproc/Makefile @@ -10,6 +10,7 @@ remoteproc-y += remoteproc_sysfs.o remoteproc-y += remoteproc_virtio.o remoteproc-y += remoteproc_elf_loader.o obj-$(CONFIG_IMX_REMOTEPROC) += imx_rproc.o +obj-$(CONFIG_MTK_SCP) += mtk_scp.o mtk_scp_ipi.o obj-$(CONFIG_OMAP_REMOTEPROC) += omap_remoteproc.o obj-$(CONFIG_WKUP_M3_RPROC) += wkup_m3_rproc.o obj-$(CONFIG_DA8XX_REMOTEPROC) += da8xx_remoteproc.o diff --git a/drivers/remoteproc/mtk_common.h b/drivers/remoteproc/mtk_common.h new file mode 100644 index 000000000000..deb20096146a --- /dev/null +++ b/drivers/remoteproc/mtk_common.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 MediaTek Inc. + */ + +#ifndef __RPROC_MTK_COMMON_H +#define __RPROC_MTK_COMMON_H + +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/platform_device.h> +#include <linux/remoteproc.h> +#include <linux/remoteproc/mtk_scp.h> + +#define MT8183_SW_RSTN 0x0 +#define MT8183_SW_RSTN_BIT BIT(0) +#define MT8183_SCP_TO_HOST 0x1C +#define MT8183_SCP_IPC_INT_BIT BIT(0) +#define MT8183_SCP_WDT_INT_BIT BIT(8) +#define MT8183_HOST_TO_SCP 0x28 +#define MT8183_HOST_IPC_INT_BIT BIT(0) +#define MT8183_WDT_CFG 0x84 +#define MT8183_SCP_CLK_SW_SEL 0x4000 +#define MT8183_SCP_CLK_DIV_SEL 0x4024 +#define MT8183_SCP_SRAM_PDN 0x402C +#define MT8183_SCP_L1_SRAM_PD 0x4080 +#define MT8183_SCP_TCM_TAIL_SRAM_PD 0x4094 + +#define MT8183_SCP_CACHE_SEL(x) (0x14000 + (x) * 0x3000) +#define MT8183_SCP_CACHE_CON MT8183_SCP_CACHE_SEL(0) +#define MT8183_SCP_DCACHE_CON MT8183_SCP_CACHE_SEL(1) +#define MT8183_SCP_CACHESIZE_8KB BIT(8) +#define MT8183_SCP_CACHE_CON_WAYEN BIT(10) + +#define SCP_FW_VER_LEN 32 +#define SCP_SHARE_BUFFER_SIZE 288 + +struct scp_run { + u32 signaled; + s8 fw_ver[SCP_FW_VER_LEN]; + u32 dec_capability; + u32 enc_capability; + wait_queue_head_t wq; +}; + +struct scp_ipi_desc { + /* For protecting handler. */ + struct mutex lock; + scp_ipi_handler_t handler; + void *priv; +}; + +struct mtk_scp { + struct device *dev; + struct rproc *rproc; + struct clk *clk; + void __iomem *reg_base; + void __iomem *sram_base; + size_t sram_size; + + struct mtk_share_obj __iomem *recv_buf; + struct mtk_share_obj __iomem *send_buf; + struct scp_run run; + /* To prevent multiple ipi_send run concurrently. */ + struct mutex send_lock; + struct scp_ipi_desc ipi_desc[SCP_IPI_MAX]; + bool ipi_id_ack[SCP_IPI_MAX]; + wait_queue_head_t ack_wq; + + void __iomem *cpu_addr; + phys_addr_t phys_addr; + size_t dram_size; + + struct rproc_subdev *rpmsg_subdev; +}; + +/** + * struct mtk_share_obj - SRAM buffer shared with AP and SCP + * + * @id: IPI id + * @len: share buffer length + * @share_buf: share buffer data + */ +struct mtk_share_obj { + u32 id; + u32 len; + u8 share_buf[SCP_SHARE_BUFFER_SIZE]; +}; + +void scp_memcpy_aligned(void __iomem *dst, const void *src, unsigned int len); +void scp_ipi_lock(struct mtk_scp *scp, u32 id); +void scp_ipi_unlock(struct mtk_scp *scp, u32 id); + +#endif diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c new file mode 100644 index 000000000000..7ccdf64ff3ea --- /dev/null +++ b/drivers/remoteproc/mtk_scp.c @@ -0,0 +1,663 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2019 MediaTek Inc. + +#include <asm/barrier.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/of_reserved_mem.h> +#include <linux/platform_device.h> +#include <linux/remoteproc.h> +#include <linux/remoteproc/mtk_scp.h> +#include <linux/rpmsg/mtk_rpmsg.h> + +#include "mtk_common.h" +#include "remoteproc_internal.h" + +#define MAX_CODE_SIZE 0x500000 +#define SCP_FW_END 0x7C000 + +/** + * scp_get() - get a reference to SCP. + * + * @pdev: the platform device of the module requesting SCP platform + * device for using SCP API. + * + * Return: Return NULL if failed. otherwise reference to SCP. + **/ +struct mtk_scp *scp_get(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *scp_node; + struct platform_device *scp_pdev; + + scp_node = of_parse_phandle(dev->of_node, "mediatek,scp", 0); + if (!scp_node) { + dev_err(dev, "can't get SCP node\n"); + return NULL; + } + + scp_pdev = of_find_device_by_node(scp_node); + of_node_put(scp_node); + + if (WARN_ON(!scp_pdev)) { + dev_err(dev, "SCP pdev failed\n"); + return NULL; + } + + return platform_get_drvdata(scp_pdev); +} +EXPORT_SYMBOL_GPL(scp_get); + +/** + * scp_put() - "free" the SCP + * + * @scp: mtk_scp structure from scp_get(). + **/ +void scp_put(struct mtk_scp *scp) +{ + put_device(scp->dev); +} +EXPORT_SYMBOL_GPL(scp_put); + +static void scp_wdt_handler(struct mtk_scp *scp, u32 scp_to_host) +{ + dev_err(scp->dev, "SCP watchdog timeout! 0x%x", scp_to_host); + rproc_report_crash(scp->rproc, RPROC_WATCHDOG); +} + +static void scp_init_ipi_handler(void *data, unsigned int len, void *priv) +{ + struct mtk_scp *scp = (struct mtk_scp *)priv; + struct scp_run *run = (struct scp_run *)data; + + scp->run.signaled = run->signaled; + strscpy(scp->run.fw_ver, run->fw_ver, SCP_FW_VER_LEN); + scp->run.dec_capability = run->dec_capability; + scp->run.enc_capability = run->enc_capability; + wake_up_interruptible(&scp->run.wq); +} + +static void scp_ipi_handler(struct mtk_scp *scp) +{ + struct mtk_share_obj __iomem *rcv_obj = scp->recv_buf; + struct scp_ipi_desc *ipi_desc = scp->ipi_desc; + u8 tmp_data[SCP_SHARE_BUFFER_SIZE]; + scp_ipi_handler_t handler; + u32 id = readl(&rcv_obj->id); + u32 len = readl(&rcv_obj->len); + + if (len > SCP_SHARE_BUFFER_SIZE) { + dev_err(scp->dev, "ipi message too long (len %d, max %d)", len, + SCP_SHARE_BUFFER_SIZE); + return; + } + if (id >= SCP_IPI_MAX) { + dev_err(scp->dev, "No such ipi id = %d\n", id); + return; + } + + scp_ipi_lock(scp, id); + handler = ipi_desc[id].handler; + if (!handler) { + dev_err(scp->dev, "No such ipi id = %d\n", id); + scp_ipi_unlock(scp, id); + return; + } + + memcpy_fromio(tmp_data, &rcv_obj->share_buf, len); + handler(tmp_data, len, ipi_desc[id].priv); + scp_ipi_unlock(scp, id); + + scp->ipi_id_ack[id] = true; + wake_up(&scp->ack_wq); +} + +static int scp_ipi_init(struct mtk_scp *scp) +{ + size_t send_offset = SCP_FW_END - sizeof(struct mtk_share_obj); + size_t recv_offset = send_offset - sizeof(struct mtk_share_obj); + + /* Disable SCP to host interrupt */ + writel(MT8183_SCP_IPC_INT_BIT, scp->reg_base + MT8183_SCP_TO_HOST); + + /* shared buffer initialization */ + scp->recv_buf = + (struct mtk_share_obj __iomem *)(scp->sram_base + recv_offset); + scp->send_buf = + (struct mtk_share_obj __iomem *)(scp->sram_base + send_offset); + memset_io(scp->recv_buf, 0, sizeof(scp->recv_buf)); + memset_io(scp->send_buf, 0, sizeof(scp->send_buf)); + + return 0; +} + +static void scp_reset_assert(const struct mtk_scp *scp) +{ + u32 val; + + val = readl(scp->reg_base + MT8183_SW_RSTN); + val &= ~MT8183_SW_RSTN_BIT; + writel(val, scp->reg_base + MT8183_SW_RSTN); +} + +static void scp_reset_deassert(const struct mtk_scp *scp) +{ + u32 val; + + val = readl(scp->reg_base + MT8183_SW_RSTN); + val |= MT8183_SW_RSTN_BIT; + writel(val, scp->reg_base + MT8183_SW_RSTN); +} + +static irqreturn_t scp_irq_handler(int irq, void *priv) +{ + struct mtk_scp *scp = priv; + u32 scp_to_host; + int ret; + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(scp->dev, "failed to enable clocks\n"); + return IRQ_NONE; + } + + scp_to_host = readl(scp->reg_base + MT8183_SCP_TO_HOST); + if (scp_to_host & MT8183_SCP_IPC_INT_BIT) + scp_ipi_handler(scp); + else + scp_wdt_handler(scp, scp_to_host); + + /* SCP won't send another interrupt until we set SCP_TO_HOST to 0. */ + writel(MT8183_SCP_IPC_INT_BIT | MT8183_SCP_WDT_INT_BIT, + scp->reg_base + MT8183_SCP_TO_HOST); + clk_disable_unprepare(scp->clk); + + return IRQ_HANDLED; +} + +static int scp_elf_load_segments(struct rproc *rproc, const struct firmware *fw) +{ + struct device *dev = &rproc->dev; + struct elf32_hdr *ehdr; + struct elf32_phdr *phdr; + int i, ret = 0; + const u8 *elf_data = fw->data; + + ehdr = (struct elf32_hdr *)elf_data; + phdr = (struct elf32_phdr *)(elf_data + ehdr->e_phoff); + + /* go through the available ELF segments */ + for (i = 0; i < ehdr->e_phnum; i++, phdr++) { + u32 da = phdr->p_paddr; + u32 memsz = phdr->p_memsz; + u32 filesz = phdr->p_filesz; + u32 offset = phdr->p_offset; + void __iomem *ptr; + + if (phdr->p_type != PT_LOAD) + continue; + + dev_dbg(dev, "phdr: type %d da 0x%x memsz 0x%x filesz 0x%x\n", + phdr->p_type, da, memsz, filesz); + + if (filesz > memsz) { + dev_err(dev, "bad phdr filesz 0x%x memsz 0x%x\n", + filesz, memsz); + ret = -EINVAL; + break; + } + + if (offset + filesz > fw->size) { + dev_err(dev, "truncated fw: need 0x%x avail 0x%zx\n", + offset + filesz, fw->size); + ret = -EINVAL; + break; + } + + /* grab the kernel address for this device address */ + ptr = (void __iomem *)rproc_da_to_va(rproc, da, memsz); + if (!ptr) { + dev_err(dev, "bad phdr da 0x%x mem 0x%x\n", da, memsz); + ret = -EINVAL; + break; + } + + /* put the segment where the remote processor expects it */ + if (phdr->p_filesz) + scp_memcpy_aligned(ptr, elf_data + phdr->p_offset, + filesz); + } + + return ret; +} + +static int scp_load(struct rproc *rproc, const struct firmware *fw) +{ + const struct mtk_scp *scp = rproc->priv; + struct device *dev = scp->dev; + int ret; + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(dev, "failed to enable clocks\n"); + return ret; + } + + /* Hold SCP in reset while loading FW. */ + scp_reset_assert(scp); + + /* Reset clocks before loading FW */ + writel(0x0, scp->reg_base + MT8183_SCP_CLK_SW_SEL); + writel(0x0, scp->reg_base + MT8183_SCP_CLK_DIV_SEL); + + /* Initialize TCM before loading FW. */ + writel(0x0, scp->reg_base + MT8183_SCP_L1_SRAM_PD); + writel(0x0, scp->reg_base + MT8183_SCP_TCM_TAIL_SRAM_PD); + + /* Turn on the power of SCP's SRAM before using it. */ + writel(0x0, scp->reg_base + MT8183_SCP_SRAM_PDN); + + /* + * Set I-cache and D-cache size before loading SCP FW. + * SCP SRAM logical address may change when cache size setting differs. + */ + writel(MT8183_SCP_CACHE_CON_WAYEN | MT8183_SCP_CACHESIZE_8KB, + scp->reg_base + MT8183_SCP_CACHE_CON); + writel(MT8183_SCP_CACHESIZE_8KB, scp->reg_base + MT8183_SCP_DCACHE_CON); + + ret = scp_elf_load_segments(rproc, fw); + clk_disable_unprepare(scp->clk); + + return ret; +} + +static int scp_start(struct rproc *rproc) +{ + struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; + struct device *dev = scp->dev; + struct scp_run *run = &scp->run; + int ret; + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(dev, "failed to enable clocks\n"); + return ret; + } + + run->signaled = false; + + scp_reset_deassert(scp); + + ret = wait_event_interruptible_timeout( + run->wq, + run->signaled, + msecs_to_jiffies(2000)); + + if (ret == 0) { + dev_err(dev, "wait SCP initialization timeout!\n"); + ret = -ETIME; + goto stop; + } + if (ret == -ERESTARTSYS) { + dev_err(dev, "wait SCP interrupted by a signal!\n"); + goto stop; + } + clk_disable_unprepare(scp->clk); + dev_info(dev, "SCP is ready. FW version %s\n", run->fw_ver); + + return 0; + +stop: + scp_reset_assert(scp); + clk_disable_unprepare(scp->clk); + return ret; +} + +static void *scp_da_to_va(struct rproc *rproc, u64 da, int len) +{ + struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; + int offset; + + if (da < scp->sram_size) { + offset = da; + if (offset >= 0 && (offset + len) < scp->sram_size) + return (void __force *)scp->sram_base + offset; + } else { + offset = da - scp->phys_addr; + if (offset >= 0 && (offset + len) < scp->dram_size) + return (void __force *)scp->cpu_addr + offset; + } + + return NULL; +} + +static int scp_stop(struct rproc *rproc) +{ + struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; + int ret; + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(scp->dev, "failed to enable clocks\n"); + return ret; + } + + scp_reset_assert(scp); + /* Disable SCP watchdog */ + writel(0, scp->reg_base + MT8183_WDT_CFG); + clk_disable_unprepare(scp->clk); + + return 0; +} + +static const struct rproc_ops scp_ops = { + .start = scp_start, + .stop = scp_stop, + .load = scp_load, + .da_to_va = scp_da_to_va, +}; + +/** + * scp_get_device() - get device struct of SCP + * + * @scp: mtk_scp structure + **/ +struct device *scp_get_device(struct mtk_scp *scp) +{ + return scp->dev; +} +EXPORT_SYMBOL_GPL(scp_get_device); + +/** + * scp_get_rproc() - get rproc struct of SCP + * + * @scp: mtk_scp structure + **/ +struct rproc *scp_get_rproc(struct mtk_scp *scp) +{ + return scp->rproc; +} +EXPORT_SYMBOL_GPL(scp_get_rproc); + +/** + * scp_get_vdec_hw_capa() - get video decoder hardware capability + * + * @scp: mtk_scp structure + * + * Return: video decoder hardware capability + **/ +unsigned int scp_get_vdec_hw_capa(struct mtk_scp *scp) +{ + return scp->run.dec_capability; +} +EXPORT_SYMBOL_GPL(scp_get_vdec_hw_capa); + +/** + * scp_get_venc_hw_capa() - get video encoder hardware capability + * + * @scp: mtk_scp structure + * + * Return: video encoder hardware capability + **/ +unsigned int scp_get_venc_hw_capa(struct mtk_scp *scp) +{ + return scp->run.enc_capability; +} +EXPORT_SYMBOL_GPL(scp_get_venc_hw_capa); + +/** + * scp_mapping_dm_addr() - Mapping SRAM/DRAM to kernel virtual address + * + * @scp: mtk_scp structure + * @mem_addr: SCP views memory address + * + * Mapping the SCP's SRAM address / + * DMEM (Data Extended Memory) memory address / + * Working buffer memory address to + * kernel virtual address. + * + * Return: Return ERR_PTR(-EINVAL) if mapping failed, + * otherwise the mapped kernel virtual address + **/ +void *scp_mapping_dm_addr(struct mtk_scp *scp, u32 mem_addr) +{ + void *ptr; + + ptr = scp_da_to_va(scp->rproc, mem_addr, 0); + if (!ptr) + return ERR_PTR(-EINVAL); + + return ptr; +} +EXPORT_SYMBOL_GPL(scp_mapping_dm_addr); + +static int scp_map_memory_region(struct mtk_scp *scp) +{ + int ret; + + ret = of_reserved_mem_device_init(scp->dev); + if (ret) { + dev_err(scp->dev, "failed to assign memory-region: %d\n", ret); + return -ENOMEM; + } + + /* Reserved SCP code size */ + scp->dram_size = MAX_CODE_SIZE; + scp->cpu_addr = dma_alloc_coherent(scp->dev, scp->dram_size, + &scp->phys_addr, GFP_KERNEL); + if (!scp->cpu_addr) + return -ENOMEM; + + return 0; +} + +static void scp_unmap_memory_region(struct mtk_scp *scp) +{ + dma_free_coherent(scp->dev, scp->dram_size, scp->cpu_addr, + scp->phys_addr); + of_reserved_mem_device_release(scp->dev); +} + +static int scp_register_ipi(struct platform_device *pdev, u32 id, + ipi_handler_t handler, void *priv) +{ + struct mtk_scp *scp = platform_get_drvdata(pdev); + + return scp_ipi_register(scp, id, handler, priv); +} + +static void scp_unregister_ipi(struct platform_device *pdev, u32 id) +{ + struct mtk_scp *scp = platform_get_drvdata(pdev); + + scp_ipi_unregister(scp, id); +} + +static int scp_send_ipi(struct platform_device *pdev, u32 id, void *buf, + unsigned int len, unsigned int wait) +{ + struct mtk_scp *scp = platform_get_drvdata(pdev); + + return scp_ipi_send(scp, id, buf, len, wait); +} + +static struct mtk_rpmsg_info mtk_scp_rpmsg_info = { + .send_ipi = scp_send_ipi, + .register_ipi = scp_register_ipi, + .unregister_ipi = scp_unregister_ipi, + .ns_ipi_id = SCP_IPI_NS_SERVICE, +}; + +static void scp_add_rpmsg_subdev(struct mtk_scp *scp) +{ + scp->rpmsg_subdev = + mtk_rpmsg_create_rproc_subdev(to_platform_device(scp->dev), + &mtk_scp_rpmsg_info); + if (scp->rpmsg_subdev) + rproc_add_subdev(scp->rproc, scp->rpmsg_subdev); +} + +static void scp_remove_rpmsg_subdev(struct mtk_scp *scp) +{ + if (scp->rpmsg_subdev) { + rproc_remove_subdev(scp->rproc, scp->rpmsg_subdev); + mtk_rpmsg_destroy_rproc_subdev(scp->rpmsg_subdev); + scp->rpmsg_subdev = NULL; + } +} + +static int scp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct mtk_scp *scp; + struct rproc *rproc; + struct resource *res; + char *fw_name = "scp.img"; + int ret, i; + + rproc = rproc_alloc(dev, + np->name, + &scp_ops, + fw_name, + sizeof(*scp)); + if (!rproc) { + dev_err(dev, "unable to allocate remoteproc\n"); + return -ENOMEM; + } + + scp = (struct mtk_scp *)rproc->priv; + scp->rproc = rproc; + scp->dev = dev; + platform_set_drvdata(pdev, scp); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + scp->sram_base = devm_ioremap_resource(dev, res); + if (IS_ERR((__force void *)scp->sram_base)) { + dev_err(dev, "Failed to parse and map sram memory\n"); + ret = PTR_ERR((__force void *)scp->sram_base); + goto free_rproc; + } + scp->sram_size = resource_size(res); + + mutex_init(&scp->send_lock); + for (i = 0; i < SCP_IPI_MAX; i++) + mutex_init(&scp->ipi_desc[i].lock); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + scp->reg_base = devm_ioremap_resource(dev, res); + if (IS_ERR((__force void *)scp->reg_base)) { + dev_err(dev, "Failed to parse and map cfg memory\n"); + ret = PTR_ERR((__force void *)scp->reg_base); + goto destroy_mutex; + } + + ret = scp_map_memory_region(scp); + if (ret) + goto destroy_mutex; + + scp->clk = devm_clk_get(dev, "main"); + if (IS_ERR(scp->clk)) { + dev_err(dev, "Failed to get clock\n"); + ret = PTR_ERR(scp->clk); + goto release_dev_mem; + } + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(dev, "failed to enable clocks\n"); + goto release_dev_mem; + } + + ret = scp_ipi_init(scp); + clk_disable_unprepare(scp->clk); + if (ret) { + dev_err(dev, "Failed to init ipi\n"); + goto release_dev_mem; + } + + /* register SCP initialization IPI */ + ret = scp_ipi_register(scp, SCP_IPI_INIT, scp_init_ipi_handler, scp); + if (ret) { + dev_err(dev, "Failed to register IPI_SCP_INIT\n"); + goto release_dev_mem; + } + + init_waitqueue_head(&scp->run.wq); + init_waitqueue_head(&scp->ack_wq); + + scp_add_rpmsg_subdev(scp); + + ret = devm_request_threaded_irq(dev, platform_get_irq(pdev, 0), NULL, + scp_irq_handler, IRQF_ONESHOT, + pdev->name, scp); + + if (ret) { + dev_err(dev, "failed to request irq\n"); + goto remove_subdev; + } + + ret = rproc_add(rproc); + if (ret) + goto remove_subdev; + + return 0; + +remove_subdev: + scp_remove_rpmsg_subdev(scp); + scp_ipi_unregister(scp, SCP_IPI_INIT); +release_dev_mem: + scp_unmap_memory_region(scp); +destroy_mutex: + for (i = 0; i < SCP_IPI_MAX; i++) + mutex_destroy(&scp->ipi_desc[i].lock); + mutex_destroy(&scp->send_lock); +free_rproc: + rproc_free(rproc); + + return ret; +} + +static int scp_remove(struct platform_device *pdev) +{ + struct mtk_scp *scp = platform_get_drvdata(pdev); + int i; + + rproc_del(scp->rproc); + scp_remove_rpmsg_subdev(scp); + scp_ipi_unregister(scp, SCP_IPI_INIT); + scp_unmap_memory_region(scp); + for (i = 0; i < SCP_IPI_MAX; i++) + mutex_destroy(&scp->ipi_desc[i].lock); + mutex_destroy(&scp->send_lock); + rproc_free(scp->rproc); + + return 0; +} + +static const struct of_device_id mtk_scp_of_match[] = { + { .compatible = "mediatek,mt8183-scp"}, + {}, +}; +MODULE_DEVICE_TABLE(of, mtk_scp_of_match); + +static struct platform_driver mtk_scp_driver = { + .probe = scp_probe, + .remove = scp_remove, + .driver = { + .name = "mtk-scp", + .of_match_table = of_match_ptr(mtk_scp_of_match), + }, +}; + +module_platform_driver(mtk_scp_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("MediaTek SCP control driver"); diff --git a/drivers/remoteproc/mtk_scp_ipi.c b/drivers/remoteproc/mtk_scp_ipi.c new file mode 100644 index 000000000000..3d3d87210ef2 --- /dev/null +++ b/drivers/remoteproc/mtk_scp_ipi.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2019 MediaTek Inc. + +#include <asm/barrier.h> +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/remoteproc/mtk_scp.h> + +#include "mtk_common.h" + +/** + * scp_ipi_register() - register an ipi function + * + * @scp: mtk_scp structure + * @id: IPI ID + * @handler: IPI handler + * @priv: private data for IPI handler + * + * Register an ipi function to receive ipi interrupt from SCP. + * + * Returns 0 if ipi registers successfully, -error on error. + */ +int scp_ipi_register(struct mtk_scp *scp, + u32 id, + scp_ipi_handler_t handler, + void *priv) +{ + if (!scp) { + dev_err(scp->dev, "scp device is not ready\n"); + return -EPROBE_DEFER; + } + + if (WARN_ON(id >= SCP_IPI_MAX) || WARN_ON(handler == NULL)) + return -EINVAL; + + scp_ipi_lock(scp, id); + scp->ipi_desc[id].handler = handler; + scp->ipi_desc[id].priv = priv; + scp_ipi_unlock(scp, id); + + return 0; +} +EXPORT_SYMBOL_GPL(scp_ipi_register); + +/** + * scp_ipi_unregister() - unregister an ipi function + * + * @scp: mtk_scp structure + * @id: IPI ID + * + * Unregister an ipi function to receive ipi interrupt from SCP. + */ +void scp_ipi_unregister(struct mtk_scp *scp, u32 id) +{ + if (!scp) + return; + + if (WARN_ON(id >= SCP_IPI_MAX)) + return; + + scp_ipi_lock(scp, id); + scp->ipi_desc[id].handler = NULL; + scp->ipi_desc[id].priv = NULL; + scp_ipi_unlock(scp, id); +} +EXPORT_SYMBOL_GPL(scp_ipi_unregister); + +/* + * scp_memcpy_aligned() - Copy src to dst, where dst is in SCP SRAM region. + * + * @dst: Pointer to the destination buffer, should be in SCP SRAM region. + * @src: Pointer to the source buffer. + * @len: Length of the source buffer to be copied. + * + * Since AP access of SCP SRAM don't support byte write, this always write a + * full word at a time, and may cause some extra bytes to be written at the + * beginning & ending of dst. + */ +void scp_memcpy_aligned(void __iomem *dst, const void *src, unsigned int len) +{ + void __iomem *ptr; + u32 val; + unsigned int i = 0, remain; + + if (!IS_ALIGNED((unsigned long)dst, 4)) { + ptr = (void __iomem *)ALIGN_DOWN((unsigned long)dst, 4); + i = 4 - (dst - ptr); + val = readl_relaxed(ptr); + memcpy((u8 *)&val + (4 - i), src, i); + writel_relaxed(val, ptr); + } + + __iowrite32_copy(dst + i, src + i, (len - i) / 4); + remain = (len - i) % 4; + + if (remain > 0) { + val = readl_relaxed(dst + len - remain); + memcpy(&val, src + len - remain, remain); + writel_relaxed(val, dst + len - remain); + } +} +EXPORT_SYMBOL_GPL(scp_memcpy_aligned); + +/** + * scp_ipi_lock() - Lock before operations of an IPI ID + * + * @scp: mtk_scp structure + * @id: IPI ID + * + * Note: This should not be used by drivers other than mtk_scp. + */ +void scp_ipi_lock(struct mtk_scp *scp, u32 id) +{ + if (WARN_ON(id >= SCP_IPI_MAX)) + return; + mutex_lock(&scp->ipi_desc[id].lock); +} +EXPORT_SYMBOL_GPL(scp_ipi_lock); + +/** + * scp_ipi_lock() - Unlock after operations of an IPI ID + * + * @scp: mtk_scp structure + * @id: IPI ID + * + * Note: This should not be used by drivers other than mtk_scp. + */ +void scp_ipi_unlock(struct mtk_scp *scp, u32 id) +{ + if (WARN_ON(id >= SCP_IPI_MAX)) + return; + mutex_unlock(&scp->ipi_desc[id].lock); +} +EXPORT_SYMBOL_GPL(scp_ipi_unlock); + +/** + * scp_ipi_send() - send data from AP to scp. + * + * @scp: mtk_scp structure + * @id: IPI ID + * @buf: the data buffer + * @len: the data buffer length + * @wait: number of msecs to wait for ack. 0 to skip waiting. + * + * This function is thread-safe. When this function returns, + * SCP has received the data and starts the processing. + * When the processing completes, IPI handler registered + * by scp_ipi_register will be called in interrupt context. + * + * Returns 0 if sending data successfully, -error on error. + **/ +int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, + unsigned int wait) +{ + struct mtk_share_obj __iomem *send_obj = scp->send_buf; + unsigned long timeout; + int ret; + + if (WARN_ON(id <= SCP_IPI_INIT) || WARN_ON(id >= SCP_IPI_MAX) || + WARN_ON(id == SCP_IPI_NS_SERVICE) || + WARN_ON(len > sizeof(send_obj->share_buf)) || WARN_ON(!buf)) + return -EINVAL; + + mutex_lock(&scp->send_lock); + + ret = clk_prepare_enable(scp->clk); + if (ret) { + dev_err(scp->dev, "failed to enable clock\n"); + goto unlock_mutex; + } + + /* Wait until SCP receives the last command */ + timeout = jiffies + msecs_to_jiffies(2000); + do { + if (time_after(jiffies, timeout)) { + dev_err(scp->dev, "%s: IPI timeout!\n", __func__); + ret = -ETIMEDOUT; + goto clock_disable; + } + } while (readl(scp->reg_base + MT8183_HOST_TO_SCP)); + + scp_memcpy_aligned(send_obj->share_buf, buf, len); + + writel(len, &send_obj->len); + writel(id, &send_obj->id); + + scp->ipi_id_ack[id] = false; + /* send the command to SCP */ + writel(MT8183_HOST_IPC_INT_BIT, scp->reg_base + MT8183_HOST_TO_SCP); + + if (wait) { + /* wait for SCP's ACK */ + timeout = msecs_to_jiffies(wait); + ret = wait_event_timeout(scp->ack_wq, + scp->ipi_id_ack[id], + timeout); + scp->ipi_id_ack[id] = false; + if (WARN(!ret, "scp ipi %d ack time out !", id)) + ret = -EIO; + else + ret = 0; + } + +clock_disable: + clk_disable_unprepare(scp->clk); +unlock_mutex: + mutex_unlock(&scp->send_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(scp_ipi_send); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("MediaTek scp IPI interface"); diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 471128a2e723..a1cc9cbe038f 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -68,14 +68,24 @@ #define AXI_HALTREQ_REG 0x0 #define AXI_HALTACK_REG 0x4 #define AXI_IDLE_REG 0x8 +#define NAV_AXI_HALTREQ_BIT BIT(0) +#define NAV_AXI_HALTACK_BIT BIT(1) +#define NAV_AXI_IDLE_BIT BIT(2) +#define AXI_GATING_VALID_OVERRIDE BIT(0) -#define HALT_ACK_TIMEOUT_MS 100 +#define HALT_ACK_TIMEOUT_US 100000 +#define NAV_HALT_ACK_TIMEOUT_US 200 /* QDSP6SS_RESET */ #define Q6SS_STOP_CORE BIT(0) #define Q6SS_CORE_ARES BIT(1) #define Q6SS_BUS_ARES_ENABLE BIT(2) +/* QDSP6SS CBCR */ +#define Q6SS_CBCR_CLKEN BIT(0) +#define Q6SS_CBCR_CLKOFF BIT(31) +#define Q6SS_CBCR_TIMEOUT_US 200 + /* QDSP6SS_GFMUX_CTL */ #define Q6SS_CLK_ENABLE BIT(1) @@ -96,15 +106,16 @@ #define QDSP6v56_BHS_ON BIT(24) #define QDSP6v56_CLAMP_WL BIT(21) #define QDSP6v56_CLAMP_QMC_MEM BIT(22) -#define HALT_CHECK_MAX_LOOPS 200 #define QDSP6SS_XO_CBCR 0x0038 #define QDSP6SS_ACC_OVERRIDE_VAL 0x20 /* QDSP6v65 parameters */ +#define QDSP6SS_CORE_CBCR 0x20 #define QDSP6SS_SLEEP 0x3C #define QDSP6SS_BOOT_CORE_START 0x400 #define QDSP6SS_BOOT_CMD 0x404 -#define SLEEP_CHECK_MAX_LOOPS 200 +#define QDSP6SS_BOOT_STATUS 0x408 +#define BOOT_STATUS_TIMEOUT_US 200 #define BOOT_FSM_TIMEOUT 10000 struct reg_info { @@ -131,6 +142,7 @@ struct rproc_hexagon_res { int version; bool need_mem_protection; bool has_alt_reset; + bool has_halt_nav; }; struct q6v5 { @@ -141,9 +153,14 @@ struct q6v5 { void __iomem *rmb_base; struct regmap *halt_map; + struct regmap *halt_nav_map; + struct regmap *conn_map; + u32 halt_q6; u32 halt_modem; u32 halt_nc; + u32 halt_nav; + u32 conn_box; struct reset_control *mss_restart; struct reset_control *pdc_reset; @@ -187,6 +204,7 @@ struct q6v5 { struct qcom_sysmon *sysmon; bool need_mem_protection; bool has_alt_reset; + bool has_halt_nav; int mpss_perm; int mba_perm; const char *hexagon_mdt_image; @@ -198,6 +216,7 @@ enum { MSS_MSM8974, MSS_MSM8996, MSS_MSM8998, + MSS_SC7180, MSS_SDM845, }; @@ -396,6 +415,26 @@ static int q6v5_reset_assert(struct q6v5 *qproc) reset_control_assert(qproc->pdc_reset); ret = reset_control_reset(qproc->mss_restart); reset_control_deassert(qproc->pdc_reset); + } else if (qproc->has_halt_nav) { + /* + * When the AXI pipeline is being reset with the Q6 modem partly + * operational there is possibility of AXI valid signal to + * glitch, leading to spurious transactions and Q6 hangs. A work + * around is employed by asserting the AXI_GATING_VALID_OVERRIDE + * BIT before triggering Q6 MSS reset. Both the HALTREQ and + * AXI_GATING_VALID_OVERRIDE are withdrawn post MSS assert + * followed by a MSS deassert, while holding the PDC reset. + */ + reset_control_assert(qproc->pdc_reset); + regmap_update_bits(qproc->conn_map, qproc->conn_box, + AXI_GATING_VALID_OVERRIDE, 1); + regmap_update_bits(qproc->halt_nav_map, qproc->halt_nav, + NAV_AXI_HALTREQ_BIT, 0); + reset_control_assert(qproc->mss_restart); + reset_control_deassert(qproc->pdc_reset); + regmap_update_bits(qproc->conn_map, qproc->conn_box, + AXI_GATING_VALID_OVERRIDE, 0); + ret = reset_control_deassert(qproc->mss_restart); } else { ret = reset_control_assert(qproc->mss_restart); } @@ -413,6 +452,8 @@ static int q6v5_reset_deassert(struct q6v5 *qproc) ret = reset_control_reset(qproc->mss_restart); writel(0, qproc->rmb_base + RMB_MBA_ALT_RESET); reset_control_deassert(qproc->pdc_reset); + } else if (qproc->has_halt_nav) { + ret = reset_control_reset(qproc->mss_restart); } else { ret = reset_control_deassert(qproc->mss_restart); } @@ -474,12 +515,12 @@ static int q6v5proc_reset(struct q6v5 *qproc) if (qproc->version == MSS_SDM845) { val = readl(qproc->reg_base + QDSP6SS_SLEEP); - val |= 0x1; + val |= Q6SS_CBCR_CLKEN; writel(val, qproc->reg_base + QDSP6SS_SLEEP); ret = readl_poll_timeout(qproc->reg_base + QDSP6SS_SLEEP, - val, !(val & BIT(31)), 1, - SLEEP_CHECK_MAX_LOOPS); + val, !(val & Q6SS_CBCR_CLKOFF), 1, + Q6SS_CBCR_TIMEOUT_US); if (ret) { dev_err(qproc->dev, "QDSP6SS Sleep clock timed out\n"); return -ETIMEDOUT; @@ -500,6 +541,54 @@ static int q6v5proc_reset(struct q6v5 *qproc) } goto pbl_wait; + } else if (qproc->version == MSS_SC7180) { + val = readl(qproc->reg_base + QDSP6SS_SLEEP); + val |= Q6SS_CBCR_CLKEN; + writel(val, qproc->reg_base + QDSP6SS_SLEEP); + + ret = readl_poll_timeout(qproc->reg_base + QDSP6SS_SLEEP, + val, !(val & Q6SS_CBCR_CLKOFF), 1, + Q6SS_CBCR_TIMEOUT_US); + if (ret) { + dev_err(qproc->dev, "QDSP6SS Sleep clock timed out\n"); + return -ETIMEDOUT; + } + + /* Turn on the XO clock needed for PLL setup */ + val = readl(qproc->reg_base + QDSP6SS_XO_CBCR); + val |= Q6SS_CBCR_CLKEN; + writel(val, qproc->reg_base + QDSP6SS_XO_CBCR); + + ret = readl_poll_timeout(qproc->reg_base + QDSP6SS_XO_CBCR, + val, !(val & Q6SS_CBCR_CLKOFF), 1, + Q6SS_CBCR_TIMEOUT_US); + if (ret) { + dev_err(qproc->dev, "QDSP6SS XO clock timed out\n"); + return -ETIMEDOUT; + } + + /* Configure Q6 core CBCR to auto-enable after reset sequence */ + val = readl(qproc->reg_base + QDSP6SS_CORE_CBCR); + val |= Q6SS_CBCR_CLKEN; + writel(val, qproc->reg_base + QDSP6SS_CORE_CBCR); + + /* De-assert the Q6 stop core signal */ + writel(1, qproc->reg_base + QDSP6SS_BOOT_CORE_START); + + /* Trigger the boot FSM to start the Q6 out-of-reset sequence */ + writel(1, qproc->reg_base + QDSP6SS_BOOT_CMD); + + /* Poll the QDSP6SS_BOOT_STATUS for FSM completion */ + ret = readl_poll_timeout(qproc->reg_base + QDSP6SS_BOOT_STATUS, + val, (val & BIT(0)) != 0, 1, + BOOT_STATUS_TIMEOUT_US); + if (ret) { + dev_err(qproc->dev, "Boot FSM failed to complete.\n"); + /* Reset the modem so that boot FSM is in reset state */ + q6v5_reset_deassert(qproc); + return ret; + } + goto pbl_wait; } else if (qproc->version == MSS_MSM8996 || qproc->version == MSS_MSM8998) { int mem_pwr_ctl; @@ -515,13 +604,13 @@ static int q6v5proc_reset(struct q6v5 *qproc) /* BHS require xo cbcr to be enabled */ val = readl(qproc->reg_base + QDSP6SS_XO_CBCR); - val |= 0x1; + val |= Q6SS_CBCR_CLKEN; writel(val, qproc->reg_base + QDSP6SS_XO_CBCR); /* Read CLKOFF bit to go low indicating CLK is enabled */ ret = readl_poll_timeout(qproc->reg_base + QDSP6SS_XO_CBCR, - val, !(val & BIT(31)), 1, - HALT_CHECK_MAX_LOOPS); + val, !(val & Q6SS_CBCR_CLKOFF), 1, + Q6SS_CBCR_TIMEOUT_US); if (ret) { dev_err(qproc->dev, "xo cbcr enabling timed out (rc:%d)\n", ret); @@ -637,7 +726,6 @@ static void q6v5proc_halt_axi_port(struct q6v5 *qproc, struct regmap *halt_map, u32 offset) { - unsigned long timeout; unsigned int val; int ret; @@ -650,14 +738,8 @@ static void q6v5proc_halt_axi_port(struct q6v5 *qproc, regmap_write(halt_map, offset + AXI_HALTREQ_REG, 1); /* Wait for halt */ - timeout = jiffies + msecs_to_jiffies(HALT_ACK_TIMEOUT_MS); - for (;;) { - ret = regmap_read(halt_map, offset + AXI_HALTACK_REG, &val); - if (ret || val || time_after(jiffies, timeout)) - break; - - msleep(1); - } + regmap_read_poll_timeout(halt_map, offset + AXI_HALTACK_REG, val, + val, 1000, HALT_ACK_TIMEOUT_US); ret = regmap_read(halt_map, offset + AXI_IDLE_REG, &val); if (ret || !val) @@ -667,6 +749,32 @@ static void q6v5proc_halt_axi_port(struct q6v5 *qproc, regmap_write(halt_map, offset + AXI_HALTREQ_REG, 0); } +static void q6v5proc_halt_nav_axi_port(struct q6v5 *qproc, + struct regmap *halt_map, + u32 offset) +{ + unsigned int val; + int ret; + + /* Check if we're already idle */ + ret = regmap_read(halt_map, offset, &val); + if (!ret && (val & NAV_AXI_IDLE_BIT)) + return; + + /* Assert halt request */ + regmap_update_bits(halt_map, offset, NAV_AXI_HALTREQ_BIT, + NAV_AXI_HALTREQ_BIT); + + /* Wait for halt ack*/ + regmap_read_poll_timeout(halt_map, offset, val, + (val & NAV_AXI_HALTACK_BIT), + 5, NAV_HALT_ACK_TIMEOUT_US); + + ret = regmap_read(halt_map, offset, &val); + if (ret || !(val & NAV_AXI_IDLE_BIT)) + dev_err(qproc->dev, "port failed halt\n"); +} + static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) { unsigned long dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS; @@ -829,6 +937,9 @@ static int q6v5_mba_load(struct q6v5 *qproc) halt_axi_ports: q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_q6); q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_modem); + if (qproc->has_halt_nav) + q6v5proc_halt_nav_axi_port(qproc, qproc->halt_nav_map, + qproc->halt_nav); q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_nc); reclaim_mba: @@ -876,6 +987,9 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc) q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_q6); q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_modem); + if (qproc->has_halt_nav) + q6v5proc_halt_nav_axi_port(qproc, qproc->halt_nav_map, + qproc->halt_nav); q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_nc); if (qproc->version == MSS_MSM8996) { /* @@ -1253,6 +1367,47 @@ static int q6v5_init_mem(struct q6v5 *qproc, struct platform_device *pdev) qproc->halt_modem = args.args[1]; qproc->halt_nc = args.args[2]; + if (qproc->has_halt_nav) { + struct platform_device *nav_pdev; + + ret = of_parse_phandle_with_fixed_args(pdev->dev.of_node, + "qcom,halt-nav-regs", + 1, 0, &args); + if (ret < 0) { + dev_err(&pdev->dev, "failed to parse halt-nav-regs\n"); + return -EINVAL; + } + + nav_pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + if (!nav_pdev) { + dev_err(&pdev->dev, "failed to get mss clock device\n"); + return -EPROBE_DEFER; + } + + qproc->halt_nav_map = dev_get_regmap(&nav_pdev->dev, NULL); + if (!qproc->halt_nav_map) { + dev_err(&pdev->dev, "failed to get map from device\n"); + return -EINVAL; + } + qproc->halt_nav = args.args[0]; + + ret = of_parse_phandle_with_fixed_args(pdev->dev.of_node, + "qcom,halt-nav-regs", + 1, 1, &args); + if (ret < 0) { + dev_err(&pdev->dev, "failed to parse halt-nav-regs\n"); + return -EINVAL; + } + + qproc->conn_map = syscon_node_to_regmap(args.np); + of_node_put(args.np); + if (IS_ERR(qproc->conn_map)) + return PTR_ERR(qproc->conn_map); + + qproc->conn_box = args.args[0]; + } + return 0; } @@ -1327,7 +1482,7 @@ static int q6v5_init_reset(struct q6v5 *qproc) return PTR_ERR(qproc->mss_restart); } - if (qproc->has_alt_reset) { + if (qproc->has_alt_reset || qproc->has_halt_nav) { qproc->pdc_reset = devm_reset_control_get_exclusive(qproc->dev, "pdc_reset"); if (IS_ERR(qproc->pdc_reset)) { @@ -1426,6 +1581,7 @@ static int q6v5_probe(struct platform_device *pdev) platform_set_drvdata(pdev, qproc); + qproc->has_halt_nav = desc->has_halt_nav; ret = q6v5_init_mem(qproc, pdev); if (ret) goto free_rproc; @@ -1549,6 +1705,41 @@ static int q6v5_remove(struct platform_device *pdev) return 0; } +static const struct rproc_hexagon_res sc7180_mss = { + .hexagon_mba_image = "mba.mbn", + .proxy_clk_names = (char*[]){ + "xo", + NULL + }, + .reset_clk_names = (char*[]){ + "iface", + "bus", + "snoc_axi", + NULL + }, + .active_clk_names = (char*[]){ + "mnoc_axi", + "nav", + "mss_nav", + "mss_crypto", + NULL + }, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "cx", + "mx", + "mss", + NULL + }, + .need_mem_protection = true, + .has_alt_reset = false, + .has_halt_nav = true, + .version = MSS_SC7180, +}; + static const struct rproc_hexagon_res sdm845_mss = { .hexagon_mba_image = "mba.mbn", .proxy_clk_names = (char*[]){ @@ -1580,6 +1771,7 @@ static const struct rproc_hexagon_res sdm845_mss = { }, .need_mem_protection = true, .has_alt_reset = true, + .has_halt_nav = false, .version = MSS_SDM845, }; @@ -1594,7 +1786,6 @@ static const struct rproc_hexagon_res msm8998_mss = { .active_clk_names = (char*[]){ "iface", "bus", - "mem", "gpll0_mss", "mnoc_axi", "snoc_axi", @@ -1607,6 +1798,7 @@ static const struct rproc_hexagon_res msm8998_mss = { }, .need_mem_protection = true, .has_alt_reset = false, + .has_halt_nav = false, .version = MSS_MSM8998, }; @@ -1636,6 +1828,7 @@ static const struct rproc_hexagon_res msm8996_mss = { }, .need_mem_protection = true, .has_alt_reset = false, + .has_halt_nav = false, .version = MSS_MSM8996, }; @@ -1668,6 +1861,7 @@ static const struct rproc_hexagon_res msm8916_mss = { }, .need_mem_protection = false, .has_alt_reset = false, + .has_halt_nav = false, .version = MSS_MSM8916, }; @@ -1708,6 +1902,7 @@ static const struct rproc_hexagon_res msm8974_mss = { }, .need_mem_protection = false, .has_alt_reset = false, + .has_halt_nav = false, .version = MSS_MSM8974, }; @@ -1717,6 +1912,7 @@ static const struct of_device_id q6v5_of_match[] = { { .compatible = "qcom,msm8974-mss-pil", .data = &msm8974_mss}, { .compatible = "qcom,msm8996-mss-pil", .data = &msm8996_mss}, { .compatible = "qcom,msm8998-mss-pil", .data = &msm8998_mss}, + { .compatible = "qcom,sc7180-mss-pil", .data = &sc7180_mss}, { .compatible = "qcom,sdm845-mss-pil", .data = &sdm845_mss}, { }, }; diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index db4b3c4bacd7..edf9d0e1a235 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -15,6 +15,8 @@ #include <linux/of_address.h> #include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> #include <linux/qcom_scm.h> #include <linux/regulator/consumer.h> #include <linux/remoteproc.h> @@ -31,6 +33,10 @@ struct adsp_data { const char *firmware_name; int pas_id; bool has_aggre2_clk; + bool auto_boot; + + char **active_pd_names; + char **proxy_pd_names; const char *ssr_name; const char *sysmon_name; @@ -49,6 +55,12 @@ struct qcom_adsp { struct regulator *cx_supply; struct regulator *px_supply; + struct device *active_pds[1]; + struct device *proxy_pds[3]; + + int active_pd_count; + int proxy_pd_count; + int pas_id; int crash_reason_smem; bool has_aggre2_clk; @@ -67,6 +79,41 @@ struct qcom_adsp { struct qcom_sysmon *sysmon; }; +static int adsp_pds_enable(struct qcom_adsp *adsp, struct device **pds, + size_t pd_count) +{ + int ret; + int i; + + for (i = 0; i < pd_count; i++) { + dev_pm_genpd_set_performance_state(pds[i], INT_MAX); + ret = pm_runtime_get_sync(pds[i]); + if (ret < 0) + goto unroll_pd_votes; + } + + return 0; + +unroll_pd_votes: + for (i--; i >= 0; i--) { + dev_pm_genpd_set_performance_state(pds[i], 0); + pm_runtime_put(pds[i]); + } + + return ret; +}; + +static void adsp_pds_disable(struct qcom_adsp *adsp, struct device **pds, + size_t pd_count) +{ + int i; + + for (i = 0; i < pd_count; i++) { + dev_pm_genpd_set_performance_state(pds[i], 0); + pm_runtime_put(pds[i]); + } +} + static int adsp_load(struct rproc *rproc, const struct firmware *fw) { struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv; @@ -84,9 +131,17 @@ static int adsp_start(struct rproc *rproc) qcom_q6v5_prepare(&adsp->q6v5); + ret = adsp_pds_enable(adsp, adsp->active_pds, adsp->active_pd_count); + if (ret < 0) + goto disable_irqs; + + ret = adsp_pds_enable(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + if (ret < 0) + goto disable_active_pds; + ret = clk_prepare_enable(adsp->xo); if (ret) - return ret; + goto disable_proxy_pds; ret = clk_prepare_enable(adsp->aggre2_clk); if (ret) @@ -124,6 +179,12 @@ disable_aggre2_clk: clk_disable_unprepare(adsp->aggre2_clk); disable_xo_clk: clk_disable_unprepare(adsp->xo); +disable_proxy_pds: + adsp_pds_disable(adsp, adsp->proxy_pds, adsp->proxy_pd_count); +disable_active_pds: + adsp_pds_disable(adsp, adsp->active_pds, adsp->active_pd_count); +disable_irqs: + qcom_q6v5_unprepare(&adsp->q6v5); return ret; } @@ -136,6 +197,7 @@ static void qcom_pas_handover(struct qcom_q6v5 *q6v5) regulator_disable(adsp->cx_supply); clk_disable_unprepare(adsp->aggre2_clk); clk_disable_unprepare(adsp->xo); + adsp_pds_disable(adsp, adsp->proxy_pds, adsp->proxy_pd_count); } static int adsp_stop(struct rproc *rproc) @@ -152,6 +214,7 @@ static int adsp_stop(struct rproc *rproc) if (ret) dev_err(adsp->dev, "failed to shutdown: %d\n", ret); + adsp_pds_disable(adsp, adsp->active_pds, adsp->active_pd_count); handover = qcom_q6v5_unprepare(&adsp->q6v5); if (handover) qcom_pas_handover(&adsp->q6v5); @@ -217,6 +280,59 @@ static int adsp_init_regulator(struct qcom_adsp *adsp) return PTR_ERR_OR_ZERO(adsp->px_supply); } +static int adsp_pds_attach(struct device *dev, struct device **devs, + char **pd_names) +{ + size_t num_pds = 0; + int ret; + int i; + + if (!pd_names) + return 0; + + /* Handle single power domain */ + if (dev->pm_domain) { + devs[0] = dev; + pm_runtime_enable(dev); + return 1; + } + + while (pd_names[num_pds]) + num_pds++; + + for (i = 0; i < num_pds; i++) { + devs[i] = dev_pm_domain_attach_by_name(dev, pd_names[i]); + if (IS_ERR_OR_NULL(devs[i])) { + ret = PTR_ERR(devs[i]) ? : -ENODATA; + goto unroll_attach; + } + } + + return num_pds; + +unroll_attach: + for (i--; i >= 0; i--) + dev_pm_domain_detach(devs[i], false); + + return ret; +}; + +static void adsp_pds_detach(struct qcom_adsp *adsp, struct device **pds, + size_t pd_count) +{ + struct device *dev = adsp->dev; + int i; + + /* Handle single power domain */ + if (dev->pm_domain && pd_count) { + pm_runtime_disable(dev); + return; + } + + for (i = 0; i < pd_count; i++) + dev_pm_domain_detach(pds[i], false); +} + static int adsp_alloc_memory_region(struct qcom_adsp *adsp) { struct device_node *node; @@ -273,6 +389,8 @@ static int adsp_probe(struct platform_device *pdev) return -ENOMEM; } + rproc->auto_boot = desc->auto_boot; + adsp = (struct qcom_adsp *)rproc->priv; adsp->dev = &pdev->dev; adsp->rproc = rproc; @@ -292,10 +410,22 @@ static int adsp_probe(struct platform_device *pdev) if (ret) goto free_rproc; + ret = adsp_pds_attach(&pdev->dev, adsp->active_pds, + desc->active_pd_names); + if (ret < 0) + goto free_rproc; + adsp->active_pd_count = ret; + + ret = adsp_pds_attach(&pdev->dev, adsp->proxy_pds, + desc->proxy_pd_names); + if (ret < 0) + goto detach_active_pds; + adsp->proxy_pd_count = ret; + ret = qcom_q6v5_init(&adsp->q6v5, pdev, rproc, desc->crash_reason_smem, qcom_pas_handover); if (ret) - goto free_rproc; + goto detach_proxy_pds; qcom_add_glink_subdev(rproc, &adsp->glink_subdev); qcom_add_smd_subdev(rproc, &adsp->smd_subdev); @@ -305,15 +435,19 @@ static int adsp_probe(struct platform_device *pdev) desc->ssctl_id); if (IS_ERR(adsp->sysmon)) { ret = PTR_ERR(adsp->sysmon); - goto free_rproc; + goto detach_proxy_pds; } ret = rproc_add(rproc); if (ret) - goto free_rproc; + goto detach_proxy_pds; return 0; +detach_proxy_pds: + adsp_pds_detach(adsp, adsp->proxy_pds, adsp->proxy_pd_count); +detach_active_pds: + adsp_pds_detach(adsp, adsp->active_pds, adsp->active_pd_count); free_rproc: rproc_free(rproc); @@ -340,6 +474,41 @@ static const struct adsp_data adsp_resource_init = { .firmware_name = "adsp.mdt", .pas_id = 1, .has_aggre2_clk = false, + .auto_boot = true, + .ssr_name = "lpass", + .sysmon_name = "adsp", + .ssctl_id = 0x14, +}; + +static const struct adsp_data sm8150_adsp_resource = { + .crash_reason_smem = 423, + .firmware_name = "adsp.mdt", + .pas_id = 1, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "cx", + NULL + }, + .ssr_name = "lpass", + .sysmon_name = "adsp", + .ssctl_id = 0x14, +}; + +static const struct adsp_data msm8998_adsp_resource = { + .crash_reason_smem = 423, + .firmware_name = "adsp.mdt", + .pas_id = 1, + .has_aggre2_clk = false, + .auto_boot = true, + .proxy_pd_names = (char*[]){ + "cx", + NULL + }, .ssr_name = "lpass", .sysmon_name = "adsp", .ssctl_id = 0x14, @@ -350,16 +519,92 @@ static const struct adsp_data cdsp_resource_init = { .firmware_name = "cdsp.mdt", .pas_id = 18, .has_aggre2_clk = false, + .auto_boot = true, + .ssr_name = "cdsp", + .sysmon_name = "cdsp", + .ssctl_id = 0x17, +}; + +static const struct adsp_data sm8150_cdsp_resource = { + .crash_reason_smem = 601, + .firmware_name = "cdsp.mdt", + .pas_id = 18, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "cx", + NULL + }, .ssr_name = "cdsp", .sysmon_name = "cdsp", .ssctl_id = 0x17, }; +static const struct adsp_data mpss_resource_init = { + .crash_reason_smem = 421, + .firmware_name = "modem.mdt", + .pas_id = 4, + .has_aggre2_clk = false, + .auto_boot = false, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "cx", + "mss", + NULL + }, + .ssr_name = "mpss", + .sysmon_name = "modem", + .ssctl_id = 0x12, +}; + static const struct adsp_data slpi_resource_init = { .crash_reason_smem = 424, .firmware_name = "slpi.mdt", .pas_id = 12, .has_aggre2_clk = true, + .auto_boot = true, + .ssr_name = "dsps", + .sysmon_name = "slpi", + .ssctl_id = 0x16, +}; + +static const struct adsp_data sm8150_slpi_resource = { + .crash_reason_smem = 424, + .firmware_name = "slpi.mdt", + .pas_id = 12, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "lcx", + "lmx", + NULL + }, + .ssr_name = "dsps", + .sysmon_name = "slpi", + .ssctl_id = 0x16, +}; + +static const struct adsp_data msm8998_slpi_resource = { + .crash_reason_smem = 424, + .firmware_name = "slpi.mdt", + .pas_id = 12, + .has_aggre2_clk = true, + .auto_boot = true, + .proxy_pd_names = (char*[]){ + "ssc_cx", + NULL + }, .ssr_name = "dsps", .sysmon_name = "slpi", .ssctl_id = 0x16, @@ -369,6 +614,7 @@ static const struct adsp_data wcss_resource_init = { .crash_reason_smem = 421, .firmware_name = "wcnss.mdt", .pas_id = 6, + .auto_boot = true, .ssr_name = "mpss", .sysmon_name = "wcnss", .ssctl_id = 0x12, @@ -378,11 +624,17 @@ static const struct of_device_id adsp_of_match[] = { { .compatible = "qcom,msm8974-adsp-pil", .data = &adsp_resource_init}, { .compatible = "qcom,msm8996-adsp-pil", .data = &adsp_resource_init}, { .compatible = "qcom,msm8996-slpi-pil", .data = &slpi_resource_init}, + { .compatible = "qcom,msm8998-adsp-pas", .data = &msm8998_adsp_resource}, + { .compatible = "qcom,msm8998-slpi-pas", .data = &msm8998_slpi_resource}, { .compatible = "qcom,qcs404-adsp-pas", .data = &adsp_resource_init }, { .compatible = "qcom,qcs404-cdsp-pas", .data = &cdsp_resource_init }, { .compatible = "qcom,qcs404-wcss-pas", .data = &wcss_resource_init }, { .compatible = "qcom,sdm845-adsp-pas", .data = &adsp_resource_init}, { .compatible = "qcom,sdm845-cdsp-pas", .data = &cdsp_resource_init}, + { .compatible = "qcom,sm8150-adsp-pas", .data = &sm8150_adsp_resource}, + { .compatible = "qcom,sm8150-cdsp-pas", .data = &sm8150_cdsp_resource}, + { .compatible = "qcom,sm8150-mpss-pas", .data = &mpss_resource_init}, + { .compatible = "qcom,sm8150-slpi-pas", .data = &sm8150_slpi_resource}, { }, }; MODULE_DEVICE_TABLE(of, adsp_of_match); diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c index c231314eab66..faf3822d8791 100644 --- a/drivers/remoteproc/qcom_sysmon.c +++ b/drivers/remoteproc/qcom_sysmon.c @@ -394,7 +394,7 @@ static int ssctl_new_server(struct qmi_handle *qmi, struct qmi_service *svc) break; default: return -EINVAL; - }; + } sysmon->ssctl_version = svc->version; diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 307df98347ba..097f33e4f1f3 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -477,8 +477,8 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc, char name[16]; /* make sure resource isn't truncated */ - if (sizeof(*rsc) + rsc->num_of_vrings * sizeof(struct fw_rsc_vdev_vring) - + rsc->config_len > avail) { + if (struct_size(rsc, vring, rsc->num_of_vrings) + rsc->config_len > + avail) { dev_err(dev, "vdev rsc is truncated\n"); return -EINVAL; } @@ -2223,7 +2223,7 @@ static int __init remoteproc_init(void) return 0; } -module_init(remoteproc_init); +subsys_initcall(remoteproc_init); static void __exit remoteproc_exit(void) { diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig index 709276540ef1..a9108ff563dc 100644 --- a/drivers/rpmsg/Kconfig +++ b/drivers/rpmsg/Kconfig @@ -15,6 +15,15 @@ config RPMSG_CHAR in /dev. They make it possible for user-space programs to send and receive rpmsg packets. +config RPMSG_MTK_SCP + tristate "MediaTek SCP" + depends on MTK_SCP + select RPMSG + help + Say y here to enable support providing communication channels to + remote processors in MediaTek platforms. + This use IPI and IPC to communicate with remote processors. + config RPMSG_QCOM_GLINK_NATIVE tristate select RPMSG diff --git a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile index 9aa859502d27..ae92a7fb08f6 100644 --- a/drivers/rpmsg/Makefile +++ b/drivers/rpmsg/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_RPMSG) += rpmsg_core.o obj-$(CONFIG_RPMSG_CHAR) += rpmsg_char.o +obj-$(CONFIG_RPMSG_MTK_SCP) += mtk_rpmsg.o obj-$(CONFIG_RPMSG_QCOM_GLINK_RPM) += qcom_glink_rpm.o obj-$(CONFIG_RPMSG_QCOM_GLINK_NATIVE) += qcom_glink_native.o obj-$(CONFIG_RPMSG_QCOM_GLINK_SMEM) += qcom_glink_smem.o diff --git a/drivers/rpmsg/mtk_rpmsg.c b/drivers/rpmsg/mtk_rpmsg.c new file mode 100644 index 000000000000..232aa4e40133 --- /dev/null +++ b/drivers/rpmsg/mtk_rpmsg.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright 2019 Google LLC. + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/remoteproc.h> +#include <linux/rpmsg/mtk_rpmsg.h> +#include <linux/workqueue.h> + +#include "rpmsg_internal.h" + +struct mtk_rpmsg_rproc_subdev { + struct platform_device *pdev; + struct mtk_rpmsg_info *info; + struct rpmsg_endpoint *ns_ept; + struct rproc_subdev subdev; + + struct work_struct register_work; + struct list_head channels; + struct mutex channels_lock; +}; + +#define to_mtk_subdev(d) container_of(d, struct mtk_rpmsg_rproc_subdev, subdev) + +struct mtk_rpmsg_channel_info { + struct rpmsg_channel_info info; + bool registered; + struct list_head list; +}; + +/** + * struct rpmsg_ns_msg - dynamic name service announcement message + * @name: name of remote service that is published + * @addr: address of remote service that is published + * + * This message is sent across to publish a new service. When we receive these + * messages, an appropriate rpmsg channel (i.e device) is created. In turn, the + * ->probe() handler of the appropriate rpmsg driver will be invoked + * (if/as-soon-as one is registered). + */ +struct rpmsg_ns_msg { + char name[RPMSG_NAME_SIZE]; + u32 addr; +} __packed; + +struct mtk_rpmsg_device { + struct rpmsg_device rpdev; + struct mtk_rpmsg_rproc_subdev *mtk_subdev; +}; + +struct mtk_rpmsg_endpoint { + struct rpmsg_endpoint ept; + struct mtk_rpmsg_rproc_subdev *mtk_subdev; +}; + +#define to_mtk_rpmsg_device(r) container_of(r, struct mtk_rpmsg_device, rpdev) +#define to_mtk_rpmsg_endpoint(r) container_of(r, struct mtk_rpmsg_endpoint, ept) + +static const struct rpmsg_endpoint_ops mtk_rpmsg_endpoint_ops; + +static void __mtk_ept_release(struct kref *kref) +{ + struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint, + refcount); + kfree(to_mtk_rpmsg_endpoint(ept)); +} + +static void mtk_rpmsg_ipi_handler(void *data, unsigned int len, void *priv) +{ + struct mtk_rpmsg_endpoint *mept = priv; + struct rpmsg_endpoint *ept = &mept->ept; + int ret; + + ret = (*ept->cb)(ept->rpdev, data, len, ept->priv, ept->addr); + if (ret) + dev_warn(&ept->rpdev->dev, "rpmsg handler return error = %d", + ret); +} + +static struct rpmsg_endpoint * +__mtk_create_ept(struct mtk_rpmsg_rproc_subdev *mtk_subdev, + struct rpmsg_device *rpdev, rpmsg_rx_cb_t cb, void *priv, + u32 id) +{ + struct mtk_rpmsg_endpoint *mept; + struct rpmsg_endpoint *ept; + struct platform_device *pdev = mtk_subdev->pdev; + int ret; + + mept = kzalloc(sizeof(*mept), GFP_KERNEL); + if (!mept) + return NULL; + mept->mtk_subdev = mtk_subdev; + + ept = &mept->ept; + kref_init(&ept->refcount); + + ept->rpdev = rpdev; + ept->cb = cb; + ept->priv = priv; + ept->ops = &mtk_rpmsg_endpoint_ops; + ept->addr = id; + + ret = mtk_subdev->info->register_ipi(pdev, id, mtk_rpmsg_ipi_handler, + mept); + if (ret) { + dev_err(&pdev->dev, "IPI register failed, id = %d", id); + kref_put(&ept->refcount, __mtk_ept_release); + return NULL; + } + + return ept; +} + +static struct rpmsg_endpoint * +mtk_rpmsg_create_ept(struct rpmsg_device *rpdev, rpmsg_rx_cb_t cb, void *priv, + struct rpmsg_channel_info chinfo) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_device(rpdev)->mtk_subdev; + + return __mtk_create_ept(mtk_subdev, rpdev, cb, priv, chinfo.src); +} + +static void mtk_rpmsg_destroy_ept(struct rpmsg_endpoint *ept) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_endpoint(ept)->mtk_subdev; + + mtk_subdev->info->unregister_ipi(mtk_subdev->pdev, ept->addr); + kref_put(&ept->refcount, __mtk_ept_release); +} + +static int mtk_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_endpoint(ept)->mtk_subdev; + + return mtk_subdev->info->send_ipi(mtk_subdev->pdev, ept->addr, data, + len, 0); +} + +static int mtk_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_endpoint(ept)->mtk_subdev; + + /* + * TODO: This currently is same as mtk_rpmsg_send, and wait until SCP + * received the last command. + */ + return mtk_subdev->info->send_ipi(mtk_subdev->pdev, ept->addr, data, + len, 0); +} + +static const struct rpmsg_endpoint_ops mtk_rpmsg_endpoint_ops = { + .destroy_ept = mtk_rpmsg_destroy_ept, + .send = mtk_rpmsg_send, + .trysend = mtk_rpmsg_trysend, +}; + +static void mtk_rpmsg_release_device(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct mtk_rpmsg_device *mdev = to_mtk_rpmsg_device(rpdev); + + kfree(mdev); +} + +static const struct rpmsg_device_ops mtk_rpmsg_device_ops = { + .create_ept = mtk_rpmsg_create_ept, +}; + +static struct device_node * +mtk_rpmsg_match_device_subnode(struct device_node *node, const char *channel) +{ + struct device_node *child; + const char *name; + int ret; + + for_each_available_child_of_node(node, child) { + ret = of_property_read_string(child, "mtk,rpmsg-name", &name); + if (ret) + continue; + + if (strcmp(name, channel) == 0) + return child; + } + + return NULL; +} + +static int mtk_rpmsg_register_device(struct mtk_rpmsg_rproc_subdev *mtk_subdev, + struct rpmsg_channel_info *info) +{ + struct rpmsg_device *rpdev; + struct mtk_rpmsg_device *mdev; + struct platform_device *pdev = mtk_subdev->pdev; + int ret; + + mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); + if (!mdev) + return -ENOMEM; + + mdev->mtk_subdev = mtk_subdev; + + rpdev = &mdev->rpdev; + rpdev->ops = &mtk_rpmsg_device_ops; + rpdev->src = info->src; + rpdev->dst = info->dst; + strscpy(rpdev->id.name, info->name, RPMSG_NAME_SIZE); + + rpdev->dev.of_node = + mtk_rpmsg_match_device_subnode(pdev->dev.of_node, info->name); + rpdev->dev.parent = &pdev->dev; + rpdev->dev.release = mtk_rpmsg_release_device; + + ret = rpmsg_register_device(rpdev); + if (ret) { + kfree(mdev); + return ret; + } + + return 0; +} + +static void mtk_register_device_work_function(struct work_struct *register_work) +{ + struct mtk_rpmsg_rproc_subdev *subdev = container_of( + register_work, struct mtk_rpmsg_rproc_subdev, register_work); + struct platform_device *pdev = subdev->pdev; + struct mtk_rpmsg_channel_info *info; + int ret; + + mutex_lock(&subdev->channels_lock); + list_for_each_entry(info, &subdev->channels, list) { + if (info->registered) + continue; + + ret = mtk_rpmsg_register_device(subdev, &info->info); + if (ret) { + dev_err(&pdev->dev, "Can't create rpmsg_device\n"); + continue; + } + + info->registered = true; + } + mutex_unlock(&subdev->channels_lock); +} + +static int mtk_rpmsg_create_device(struct mtk_rpmsg_rproc_subdev *mtk_subdev, + char *name, u32 addr) +{ + struct mtk_rpmsg_channel_info *info; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + strscpy(info->info.name, name, RPMSG_NAME_SIZE); + info->info.src = addr; + info->info.dst = RPMSG_ADDR_ANY; + mutex_lock(&mtk_subdev->channels_lock); + list_add(&info->list, &mtk_subdev->channels); + mutex_unlock(&mtk_subdev->channels_lock); + + schedule_work(&mtk_subdev->register_work); + return 0; +} + +static int mtk_rpmsg_ns_cb(struct rpmsg_device *rpdev, void *data, int len, + void *priv, u32 src) +{ + struct rpmsg_ns_msg *msg = data; + struct mtk_rpmsg_rproc_subdev *mtk_subdev = priv; + struct device *dev = &mtk_subdev->pdev->dev; + + int ret; + + if (len != sizeof(*msg)) { + dev_err(dev, "malformed ns msg (%d)\n", len); + return -EINVAL; + } + + /* + * the name service ept does _not_ belong to a real rpmsg channel, + * and is handled by the rpmsg bus itself. + * for sanity reasons, make sure a valid rpdev has _not_ sneaked + * in somehow. + */ + if (rpdev) { + dev_err(dev, "anomaly: ns ept has an rpdev handle\n"); + return -EINVAL; + } + + /* don't trust the remote processor for null terminating the name */ + msg->name[RPMSG_NAME_SIZE - 1] = '\0'; + + dev_info(dev, "creating channel %s addr 0x%x\n", msg->name, msg->addr); + + ret = mtk_rpmsg_create_device(mtk_subdev, msg->name, msg->addr); + if (ret) { + dev_err(dev, "create rpmsg device failed\n"); + return ret; + } + + return 0; +} + +static int mtk_rpmsg_prepare(struct rproc_subdev *subdev) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + + /* a dedicated endpoint handles the name service msgs */ + if (mtk_subdev->info->ns_ipi_id >= 0) { + mtk_subdev->ns_ept = + __mtk_create_ept(mtk_subdev, NULL, mtk_rpmsg_ns_cb, + mtk_subdev, + mtk_subdev->info->ns_ipi_id); + if (!mtk_subdev->ns_ept) { + dev_err(&mtk_subdev->pdev->dev, + "failed to create name service endpoint\n"); + return -ENOMEM; + } + } + + return 0; +} + +static void mtk_rpmsg_unprepare(struct rproc_subdev *subdev) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + + if (mtk_subdev->ns_ept) { + mtk_rpmsg_destroy_ept(mtk_subdev->ns_ept); + mtk_subdev->ns_ept = NULL; + } +} + +static void mtk_rpmsg_stop(struct rproc_subdev *subdev, bool crashed) +{ + struct mtk_rpmsg_channel_info *info, *next; + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + struct device *dev = &mtk_subdev->pdev->dev; + + /* + * Destroy the name service endpoint here, to avoid new channel being + * created after the rpmsg_unregister_device loop below. + */ + if (mtk_subdev->ns_ept) { + mtk_rpmsg_destroy_ept(mtk_subdev->ns_ept); + mtk_subdev->ns_ept = NULL; + } + + cancel_work_sync(&mtk_subdev->register_work); + + mutex_lock(&mtk_subdev->channels_lock); + list_for_each_entry(info, &mtk_subdev->channels, list) { + if (!info->registered) + continue; + if (rpmsg_unregister_device(dev, &info->info)) { + dev_warn( + dev, + "rpmsg_unregister_device failed for %s.%d.%d\n", + info->info.name, info->info.src, + info->info.dst); + } + } + + list_for_each_entry_safe(info, next, + &mtk_subdev->channels, list) { + list_del(&info->list); + kfree(info); + } + mutex_unlock(&mtk_subdev->channels_lock); +} + +struct rproc_subdev * +mtk_rpmsg_create_rproc_subdev(struct platform_device *pdev, + struct mtk_rpmsg_info *info) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev; + + mtk_subdev = kzalloc(sizeof(*mtk_subdev), GFP_KERNEL); + if (!mtk_subdev) + return NULL; + + mtk_subdev->pdev = pdev; + mtk_subdev->subdev.prepare = mtk_rpmsg_prepare; + mtk_subdev->subdev.stop = mtk_rpmsg_stop; + mtk_subdev->subdev.unprepare = mtk_rpmsg_unprepare; + mtk_subdev->info = info; + INIT_LIST_HEAD(&mtk_subdev->channels); + INIT_WORK(&mtk_subdev->register_work, + mtk_register_device_work_function); + mutex_init(&mtk_subdev->channels_lock); + + return &mtk_subdev->subdev; +} +EXPORT_SYMBOL_GPL(mtk_rpmsg_create_rproc_subdev); + +void mtk_rpmsg_destroy_rproc_subdev(struct rproc_subdev *subdev) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + + kfree(mtk_subdev); +} +EXPORT_SYMBOL_GPL(mtk_rpmsg_destroy_rproc_subdev); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("MediaTek scp rpmsg driver"); diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index d77515d8382c..34c8b6c7e095 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -240,6 +240,7 @@ config RTC_DRV_AS3722 config RTC_DRV_DS1307 tristate "Dallas/Maxim DS1307/37/38/39/40/41, ST M41T00, EPSON RX-8025, ISL12057" + select REGMAP_I2C help If you say yes here you get support for various compatible RTC chips (often with battery backup) connected with I2C. This driver @@ -498,12 +499,13 @@ config RTC_DRV_M41T80_WDT help If you say Y here you will get support for the watchdog timer in the ST M41T60 and M41T80 RTC chips series. + config RTC_DRV_BD70528 tristate "ROHM BD70528 PMIC RTC" depends on MFD_ROHM_BD70528 && (BD70528_WATCHDOG || !BD70528_WATCHDOG) help If you say Y here you will get support for the RTC - on ROHM BD70528 Power Management IC. + block on ROHM BD70528 and BD71828 Power Management IC. This driver can also be built as a module. If so, the module will be called rtc-bd70528. @@ -621,6 +623,7 @@ config RTC_DRV_RX8010 config RTC_DRV_RX8581 tristate "Epson RX-8571/RX-8581" + select REGMAP_I2C help If you say yes here you will get support for the Epson RX-8571/ RX-8581. @@ -648,6 +651,7 @@ config RTC_DRV_EM3027 config RTC_DRV_RV3028 tristate "Micro Crystal RV3028" + select REGMAP_I2C help If you say yes here you get support for the Micro Crystal RV3028. @@ -676,13 +680,14 @@ config RTC_DRV_S5M will be called rtc-s5m. config RTC_DRV_SD3078 - tristate "ZXW Shenzhen whwave SD3078" - help - If you say yes here you get support for the ZXW Shenzhen whwave - SD3078 RTC chips. + tristate "ZXW Shenzhen whwave SD3078" + select REGMAP_I2C + help + If you say yes here you get support for the ZXW Shenzhen whwave + SD3078 RTC chips. - This driver can also be built as a module. If so, the module - will be called rtc-sd3078 + This driver can also be built as a module. If so, the module + will be called rtc-sd3078 endif # I2C @@ -848,14 +853,14 @@ config RTC_I2C_AND_SPI default m if I2C=m default y if I2C=y default y if SPI_MASTER=y - select REGMAP_I2C if I2C - select REGMAP_SPI if SPI_MASTER comment "SPI and I2C RTC drivers" config RTC_DRV_DS3232 tristate "Dallas/Maxim DS3232/DS3234" depends on RTC_I2C_AND_SPI + select REGMAP_I2C if I2C + select REGMAP_SPI if SPI_MASTER help If you say yes here you get support for Dallas Semiconductor DS3232 and DS3234 real-time clock chips. If an interrupt is associated @@ -875,6 +880,8 @@ config RTC_DRV_DS3232_HWMON config RTC_DRV_PCF2127 tristate "NXP PCF2127" depends on RTC_I2C_AND_SPI + select REGMAP_I2C if I2C + select REGMAP_SPI if SPI_MASTER select WATCHDOG_CORE if WATCHDOG help If you say yes here you get support for the NXP PCF2127/29 RTC @@ -891,6 +898,8 @@ config RTC_DRV_PCF2127 config RTC_DRV_RV3029C2 tristate "Micro Crystal RV3029/3049" depends on RTC_I2C_AND_SPI + select REGMAP_I2C if I2C + select REGMAP_SPI if SPI_MASTER help If you say yes here you get support for the Micro Crystal RV3029 and RV3049 RTC chips. diff --git a/drivers/rtc/rtc-abx80x.c b/drivers/rtc/rtc-abx80x.c index 73830670a41f..3521d8e8dc38 100644 --- a/drivers/rtc/rtc-abx80x.c +++ b/drivers/rtc/rtc-abx80x.c @@ -523,12 +523,9 @@ static int abx80x_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) if (status < 0) return status; - tmp = !!(status & ABX8XX_STATUS_BLF); + tmp = status & ABX8XX_STATUS_BLF ? RTC_VL_BACKUP_LOW : 0; - if (copy_to_user((void __user *)arg, &tmp, sizeof(int))) - return -EFAULT; - - return 0; + return put_user(tmp, (unsigned int __user *)arg); case RTC_VL_CLR: status = i2c_smbus_read_byte_data(client, ABX8XX_REG_STATUS); diff --git a/drivers/rtc/rtc-asm9260.c b/drivers/rtc/rtc-asm9260.c index 10064bdabdff..3ab81cdec00b 100644 --- a/drivers/rtc/rtc-asm9260.c +++ b/drivers/rtc/rtc-asm9260.c @@ -264,6 +264,9 @@ static int asm9260_rtc_probe(struct platform_device *pdev) return PTR_ERR(priv->iobase); priv->clk = devm_clk_get(dev, "ahb"); + if (IS_ERR(priv->clk)) + return PTR_ERR(priv->clk); + ret = clk_prepare_enable(priv->clk); if (ret) { dev_err(dev, "Failed to enable clk!\n"); diff --git a/drivers/rtc/rtc-at91rm9200.c b/drivers/rtc/rtc-at91rm9200.c index 3b833e02a657..5e811e04cb21 100644 --- a/drivers/rtc/rtc-at91rm9200.c +++ b/drivers/rtc/rtc-at91rm9200.c @@ -14,6 +14,7 @@ */ #include <linux/bcd.h> +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/completion.h> #include <linux/interrupt.h> @@ -30,7 +31,51 @@ #include <linux/time.h> #include <linux/uaccess.h> -#include "rtc-at91rm9200.h" +#define AT91_RTC_CR 0x00 /* Control Register */ +#define AT91_RTC_UPDTIM BIT(0) /* Update Request Time Register */ +#define AT91_RTC_UPDCAL BIT(1) /* Update Request Calendar Register */ + +#define AT91_RTC_MR 0x04 /* Mode Register */ + +#define AT91_RTC_TIMR 0x08 /* Time Register */ +#define AT91_RTC_SEC GENMASK(6, 0) /* Current Second */ +#define AT91_RTC_MIN GENMASK(14, 8) /* Current Minute */ +#define AT91_RTC_HOUR GENMASK(21, 16) /* Current Hour */ +#define AT91_RTC_AMPM BIT(22) /* Ante Meridiem Post Meridiem Indicator */ + +#define AT91_RTC_CALR 0x0c /* Calendar Register */ +#define AT91_RTC_CENT GENMASK(6, 0) /* Current Century */ +#define AT91_RTC_YEAR GENMASK(15, 8) /* Current Year */ +#define AT91_RTC_MONTH GENMASK(20, 16) /* Current Month */ +#define AT91_RTC_DAY GENMASK(23, 21) /* Current Day */ +#define AT91_RTC_DATE GENMASK(29, 24) /* Current Date */ + +#define AT91_RTC_TIMALR 0x10 /* Time Alarm Register */ +#define AT91_RTC_SECEN BIT(7) /* Second Alarm Enable */ +#define AT91_RTC_MINEN BIT(15) /* Minute Alarm Enable */ +#define AT91_RTC_HOUREN BIT(23) /* Hour Alarm Enable */ + +#define AT91_RTC_CALALR 0x14 /* Calendar Alarm Register */ +#define AT91_RTC_MTHEN BIT(23) /* Month Alarm Enable */ +#define AT91_RTC_DATEEN BIT(31) /* Date Alarm Enable */ + +#define AT91_RTC_SR 0x18 /* Status Register */ +#define AT91_RTC_ACKUPD BIT(0) /* Acknowledge for Update */ +#define AT91_RTC_ALARM BIT(1) /* Alarm Flag */ +#define AT91_RTC_SECEV BIT(2) /* Second Event */ +#define AT91_RTC_TIMEV BIT(3) /* Time Event */ +#define AT91_RTC_CALEV BIT(4) /* Calendar Event */ + +#define AT91_RTC_SCCR 0x1c /* Status Clear Command Register */ +#define AT91_RTC_IER 0x20 /* Interrupt Enable Register */ +#define AT91_RTC_IDR 0x24 /* Interrupt Disable Register */ +#define AT91_RTC_IMR 0x28 /* Interrupt Mask Register */ + +#define AT91_RTC_VER 0x2c /* Valid Entry Register */ +#define AT91_RTC_NVTIM BIT(0) /* Non valid Time */ +#define AT91_RTC_NVCAL BIT(1) /* Non valid Calendar */ +#define AT91_RTC_NVTIMALR BIT(2) /* Non valid Time Alarm */ +#define AT91_RTC_NVCALALR BIT(3) /* Non valid Calendar Alarm */ #define at91_rtc_read(field) \ readl_relaxed(at91_rtc_regs + field) @@ -117,20 +162,20 @@ static void at91_rtc_decodetime(unsigned int timereg, unsigned int calreg, } while ((time != at91_rtc_read(timereg)) || (date != at91_rtc_read(calreg))); - tm->tm_sec = bcd2bin((time & AT91_RTC_SEC) >> 0); - tm->tm_min = bcd2bin((time & AT91_RTC_MIN) >> 8); - tm->tm_hour = bcd2bin((time & AT91_RTC_HOUR) >> 16); + tm->tm_sec = bcd2bin(FIELD_GET(AT91_RTC_SEC, time)); + tm->tm_min = bcd2bin(FIELD_GET(AT91_RTC_MIN, time)); + tm->tm_hour = bcd2bin(FIELD_GET(AT91_RTC_HOUR, time)); /* * The Calendar Alarm register does not have a field for * the year - so these will return an invalid value. */ tm->tm_year = bcd2bin(date & AT91_RTC_CENT) * 100; /* century */ - tm->tm_year += bcd2bin((date & AT91_RTC_YEAR) >> 8); /* year */ + tm->tm_year += bcd2bin(FIELD_GET(AT91_RTC_YEAR, date)); /* year */ - tm->tm_wday = bcd2bin((date & AT91_RTC_DAY) >> 21) - 1; /* day of the week [0-6], Sunday=0 */ - tm->tm_mon = bcd2bin((date & AT91_RTC_MONTH) >> 16) - 1; - tm->tm_mday = bcd2bin((date & AT91_RTC_DATE) >> 24); + tm->tm_wday = bcd2bin(FIELD_GET(AT91_RTC_DAY, date)) - 1; /* day of the week [0-6], Sunday=0 */ + tm->tm_mon = bcd2bin(FIELD_GET(AT91_RTC_MONTH, date)) - 1; + tm->tm_mday = bcd2bin(FIELD_GET(AT91_RTC_DATE, date)); } /* @@ -167,16 +212,17 @@ static int at91_rtc_settime(struct device *dev, struct rtc_time *tm) at91_rtc_write_idr(AT91_RTC_ACKUPD); at91_rtc_write(AT91_RTC_TIMR, - bin2bcd(tm->tm_sec) << 0 - | bin2bcd(tm->tm_min) << 8 - | bin2bcd(tm->tm_hour) << 16); + FIELD_PREP(AT91_RTC_SEC, bin2bcd(tm->tm_sec)) + | FIELD_PREP(AT91_RTC_MIN, bin2bcd(tm->tm_min)) + | FIELD_PREP(AT91_RTC_HOUR, bin2bcd(tm->tm_hour))); at91_rtc_write(AT91_RTC_CALR, - bin2bcd((tm->tm_year + 1900) / 100) /* century */ - | bin2bcd(tm->tm_year % 100) << 8 /* year */ - | bin2bcd(tm->tm_mon + 1) << 16 /* tm_mon starts at zero */ - | bin2bcd(tm->tm_wday + 1) << 21 /* day of the week [0-6], Sunday=0 */ - | bin2bcd(tm->tm_mday) << 24); + FIELD_PREP(AT91_RTC_CENT, + bin2bcd((tm->tm_year + 1900) / 100)) + | FIELD_PREP(AT91_RTC_YEAR, bin2bcd(tm->tm_year % 100)) + | FIELD_PREP(AT91_RTC_MONTH, bin2bcd(tm->tm_mon + 1)) + | FIELD_PREP(AT91_RTC_DAY, bin2bcd(tm->tm_wday + 1)) + | FIELD_PREP(AT91_RTC_DATE, bin2bcd(tm->tm_mday))); /* Restart Time/Calendar */ cr = at91_rtc_read(AT91_RTC_CR); @@ -211,25 +257,17 @@ static int at91_rtc_readalarm(struct device *dev, struct rtc_wkalrm *alrm) */ static int at91_rtc_setalarm(struct device *dev, struct rtc_wkalrm *alrm) { - struct rtc_time tm; - - at91_rtc_decodetime(AT91_RTC_TIMR, AT91_RTC_CALR, &tm); - - tm.tm_mon = alrm->time.tm_mon; - tm.tm_mday = alrm->time.tm_mday; - tm.tm_hour = alrm->time.tm_hour; - tm.tm_min = alrm->time.tm_min; - tm.tm_sec = alrm->time.tm_sec; + struct rtc_time tm = alrm->time; at91_rtc_write_idr(AT91_RTC_ALARM); at91_rtc_write(AT91_RTC_TIMALR, - bin2bcd(tm.tm_sec) << 0 - | bin2bcd(tm.tm_min) << 8 - | bin2bcd(tm.tm_hour) << 16 + FIELD_PREP(AT91_RTC_SEC, bin2bcd(alrm->time.tm_sec)) + | FIELD_PREP(AT91_RTC_MIN, bin2bcd(alrm->time.tm_min)) + | FIELD_PREP(AT91_RTC_HOUR, bin2bcd(alrm->time.tm_hour)) | AT91_RTC_HOUREN | AT91_RTC_MINEN | AT91_RTC_SECEN); at91_rtc_write(AT91_RTC_CALALR, - bin2bcd(tm.tm_mon + 1) << 16 /* tm_mon starts at zero */ - | bin2bcd(tm.tm_mday) << 24 + FIELD_PREP(AT91_RTC_MONTH, bin2bcd(alrm->time.tm_mon + 1)) + | FIELD_PREP(AT91_RTC_DATE, bin2bcd(alrm->time.tm_mday)) | AT91_RTC_DATEEN | AT91_RTC_MTHEN); if (alrm->enabled) { @@ -254,20 +292,6 @@ static int at91_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled) return 0; } -/* - * Provide additional RTC information in /proc/driver/rtc - */ -static int at91_rtc_proc(struct device *dev, struct seq_file *seq) -{ - unsigned long imr = at91_rtc_read_imr(); - - seq_printf(seq, "update_IRQ\t: %s\n", - (imr & AT91_RTC_ACKUPD) ? "yes" : "no"); - seq_printf(seq, "periodic_IRQ\t: %s\n", - (imr & AT91_RTC_SECEV) ? "yes" : "no"); - - return 0; -} /* * IRQ handler for the RTC @@ -327,6 +351,12 @@ static const struct of_device_id at91_rtc_dt_ids[] = { .compatible = "atmel,at91sam9x5-rtc", .data = &at91sam9x5_config, }, { + .compatible = "atmel,sama5d4-rtc", + .data = &at91rm9200_config, + }, { + .compatible = "atmel,sama5d2-rtc", + .data = &at91rm9200_config, + }, { /* sentinel */ } }; @@ -337,7 +367,6 @@ static const struct rtc_class_ops at91_rtc_ops = { .set_time = at91_rtc_settime, .read_alarm = at91_rtc_readalarm, .set_alarm = at91_rtc_setalarm, - .proc = at91_rtc_proc, .alarm_irq_enable = at91_rtc_alarm_irq_enable, }; diff --git a/drivers/rtc/rtc-at91rm9200.h b/drivers/rtc/rtc-at91rm9200.h deleted file mode 100644 index 8be5289da8e2..000000000000 --- a/drivers/rtc/rtc-at91rm9200.h +++ /dev/null @@ -1,71 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * arch/arm/mach-at91/include/mach/at91_rtc.h - * - * Copyright (C) 2005 Ivan Kokshaysky - * Copyright (C) SAN People - * - * Real Time Clock (RTC) - System peripheral registers. - * Based on AT91RM9200 datasheet revision E. - */ - -#ifndef AT91_RTC_H -#define AT91_RTC_H - -#define AT91_RTC_CR 0x00 /* Control Register */ -#define AT91_RTC_UPDTIM (1 << 0) /* Update Request Time Register */ -#define AT91_RTC_UPDCAL (1 << 1) /* Update Request Calendar Register */ -#define AT91_RTC_TIMEVSEL (3 << 8) /* Time Event Selection */ -#define AT91_RTC_TIMEVSEL_MINUTE (0 << 8) -#define AT91_RTC_TIMEVSEL_HOUR (1 << 8) -#define AT91_RTC_TIMEVSEL_DAY24 (2 << 8) -#define AT91_RTC_TIMEVSEL_DAY12 (3 << 8) -#define AT91_RTC_CALEVSEL (3 << 16) /* Calendar Event Selection */ -#define AT91_RTC_CALEVSEL_WEEK (0 << 16) -#define AT91_RTC_CALEVSEL_MONTH (1 << 16) -#define AT91_RTC_CALEVSEL_YEAR (2 << 16) - -#define AT91_RTC_MR 0x04 /* Mode Register */ -#define AT91_RTC_HRMOD (1 << 0) /* 12/24 Hour Mode */ - -#define AT91_RTC_TIMR 0x08 /* Time Register */ -#define AT91_RTC_SEC (0x7f << 0) /* Current Second */ -#define AT91_RTC_MIN (0x7f << 8) /* Current Minute */ -#define AT91_RTC_HOUR (0x3f << 16) /* Current Hour */ -#define AT91_RTC_AMPM (1 << 22) /* Ante Meridiem Post Meridiem Indicator */ - -#define AT91_RTC_CALR 0x0c /* Calendar Register */ -#define AT91_RTC_CENT (0x7f << 0) /* Current Century */ -#define AT91_RTC_YEAR (0xff << 8) /* Current Year */ -#define AT91_RTC_MONTH (0x1f << 16) /* Current Month */ -#define AT91_RTC_DAY (7 << 21) /* Current Day */ -#define AT91_RTC_DATE (0x3f << 24) /* Current Date */ - -#define AT91_RTC_TIMALR 0x10 /* Time Alarm Register */ -#define AT91_RTC_SECEN (1 << 7) /* Second Alarm Enable */ -#define AT91_RTC_MINEN (1 << 15) /* Minute Alarm Enable */ -#define AT91_RTC_HOUREN (1 << 23) /* Hour Alarm Enable */ - -#define AT91_RTC_CALALR 0x14 /* Calendar Alarm Register */ -#define AT91_RTC_MTHEN (1 << 23) /* Month Alarm Enable */ -#define AT91_RTC_DATEEN (1 << 31) /* Date Alarm Enable */ - -#define AT91_RTC_SR 0x18 /* Status Register */ -#define AT91_RTC_ACKUPD (1 << 0) /* Acknowledge for Update */ -#define AT91_RTC_ALARM (1 << 1) /* Alarm Flag */ -#define AT91_RTC_SECEV (1 << 2) /* Second Event */ -#define AT91_RTC_TIMEV (1 << 3) /* Time Event */ -#define AT91_RTC_CALEV (1 << 4) /* Calendar Event */ - -#define AT91_RTC_SCCR 0x1c /* Status Clear Command Register */ -#define AT91_RTC_IER 0x20 /* Interrupt Enable Register */ -#define AT91_RTC_IDR 0x24 /* Interrupt Disable Register */ -#define AT91_RTC_IMR 0x28 /* Interrupt Mask Register */ - -#define AT91_RTC_VER 0x2c /* Valid Entry Register */ -#define AT91_RTC_NVTIM (1 << 0) /* Non valid Time */ -#define AT91_RTC_NVCAL (1 << 1) /* Non valid Calendar */ -#define AT91_RTC_NVTIMALR (1 << 2) /* Non valid Time Alarm */ -#define AT91_RTC_NVCALALR (1 << 3) /* Non valid Calendar Alarm */ - -#endif diff --git a/drivers/rtc/rtc-bd70528.c b/drivers/rtc/rtc-bd70528.c index 627037aa66a8..bbbb1f07c91f 100644 --- a/drivers/rtc/rtc-bd70528.c +++ b/drivers/rtc/rtc-bd70528.c @@ -6,6 +6,7 @@ #include <linux/bcd.h> #include <linux/mfd/rohm-bd70528.h> +#include <linux/mfd/rohm-bd71828.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> @@ -15,7 +16,7 @@ /* * We read regs RTC_SEC => RTC_YEAR * this struct is ordered according to chip registers. - * Keep it u8 only to avoid padding issues. + * Keep it u8 only (or packed) to avoid padding issues. */ struct bd70528_rtc_day { u8 sec; @@ -36,6 +37,13 @@ struct bd70528_rtc_wake { u8 ctrl; } __packed; +struct bd71828_rtc_alm { + struct bd70528_rtc_data alm0; + struct bd70528_rtc_data alm1; + u8 alm_mask; + u8 alm1_mask; +} __packed; + struct bd70528_rtc_alm { struct bd70528_rtc_data data; u8 alm_mask; @@ -43,8 +51,10 @@ struct bd70528_rtc_alm { } __packed; struct bd70528_rtc { - struct rohm_regmap_dev *mfd; + struct rohm_regmap_dev *parent; struct device *dev; + u8 reg_time_start; + bool has_rtc_timers; }; static int bd70528_set_wake(struct rohm_regmap_dev *bd70528, @@ -123,14 +133,14 @@ static int bd70528_set_rtc_based_timers(struct bd70528_rtc *r, int new_state, { int ret; - ret = bd70528_wdt_set(r->mfd, new_state & BD70528_WDT_STATE_BIT, + ret = bd70528_wdt_set(r->parent, new_state & BD70528_WDT_STATE_BIT, old_state); if (ret) { dev_err(r->dev, "Failed to disable WDG for RTC setting (%d)\n", ret); return ret; } - ret = bd70528_set_elapsed_tmr(r->mfd, + ret = bd70528_set_elapsed_tmr(r->parent, new_state & BD70528_ELAPSED_STATE_BIT, old_state); if (ret) { @@ -138,7 +148,7 @@ static int bd70528_set_rtc_based_timers(struct bd70528_rtc *r, int new_state, "Failed to disable 'elapsed timer' for RTC setting\n"); return ret; } - ret = bd70528_set_wake(r->mfd, new_state & BD70528_WAKE_STATE_BIT, + ret = bd70528_set_wake(r->parent, new_state & BD70528_WAKE_STATE_BIT, old_state); if (ret) { dev_err(r->dev, @@ -152,12 +162,18 @@ static int bd70528_set_rtc_based_timers(struct bd70528_rtc *r, int new_state, static int bd70528_re_enable_rtc_based_timers(struct bd70528_rtc *r, int old_state) { + if (!r->has_rtc_timers) + return 0; + return bd70528_set_rtc_based_timers(r, old_state, NULL); } static int bd70528_disable_rtc_based_timers(struct bd70528_rtc *r, int *old_state) { + if (!r->has_rtc_timers) + return 0; + return bd70528_set_rtc_based_timers(r, 0, old_state); } @@ -213,22 +229,52 @@ static inline void rtc2tm(struct bd70528_rtc_data *r, struct rtc_time *t) t->tm_wday = bcd2bin(r->week & BD70528_MASK_RTC_WEEK); } +static int bd71828_set_alarm(struct device *dev, struct rtc_wkalrm *a) +{ + int ret; + struct bd71828_rtc_alm alm; + struct bd70528_rtc *r = dev_get_drvdata(dev); + struct rohm_regmap_dev *parent = r->parent; + + ret = regmap_bulk_read(parent->regmap, BD71828_REG_RTC_ALM_START, + &alm, sizeof(alm)); + if (ret) { + dev_err(dev, "Failed to read alarm regs\n"); + return ret; + } + + tm2rtc(&a->time, &alm.alm0); + + if (!a->enabled) + alm.alm_mask &= ~BD70528_MASK_ALM_EN; + else + alm.alm_mask |= BD70528_MASK_ALM_EN; + + ret = regmap_bulk_write(parent->regmap, BD71828_REG_RTC_ALM_START, + &alm, sizeof(alm)); + if (ret) + dev_err(dev, "Failed to set alarm time\n"); + + return ret; + +} + static int bd70528_set_alarm(struct device *dev, struct rtc_wkalrm *a) { struct bd70528_rtc_wake wake; struct bd70528_rtc_alm alm; int ret; struct bd70528_rtc *r = dev_get_drvdata(dev); - struct rohm_regmap_dev *bd70528 = r->mfd; + struct rohm_regmap_dev *parent = r->parent; - ret = regmap_bulk_read(bd70528->regmap, BD70528_REG_RTC_WAKE_START, + ret = regmap_bulk_read(parent->regmap, BD70528_REG_RTC_WAKE_START, &wake, sizeof(wake)); if (ret) { dev_err(dev, "Failed to read wake regs\n"); return ret; } - ret = regmap_bulk_read(bd70528->regmap, BD70528_REG_RTC_ALM_START, + ret = regmap_bulk_read(parent->regmap, BD70528_REG_RTC_ALM_START, &alm, sizeof(alm)); if (ret) { dev_err(dev, "Failed to read alarm regs\n"); @@ -246,14 +292,14 @@ static int bd70528_set_alarm(struct device *dev, struct rtc_wkalrm *a) wake.ctrl &= ~BD70528_MASK_WAKE_EN; } - ret = regmap_bulk_write(bd70528->regmap, + ret = regmap_bulk_write(parent->regmap, BD70528_REG_RTC_WAKE_START, &wake, sizeof(wake)); if (ret) { dev_err(dev, "Failed to set wake time\n"); return ret; } - ret = regmap_bulk_write(bd70528->regmap, BD70528_REG_RTC_ALM_START, + ret = regmap_bulk_write(parent->regmap, BD70528_REG_RTC_ALM_START, &alm, sizeof(alm)); if (ret) dev_err(dev, "Failed to set alarm time\n"); @@ -261,14 +307,38 @@ static int bd70528_set_alarm(struct device *dev, struct rtc_wkalrm *a) return ret; } +static int bd71828_read_alarm(struct device *dev, struct rtc_wkalrm *a) +{ + int ret; + struct bd71828_rtc_alm alm; + struct bd70528_rtc *r = dev_get_drvdata(dev); + struct rohm_regmap_dev *parent = r->parent; + + ret = regmap_bulk_read(parent->regmap, BD71828_REG_RTC_ALM_START, + &alm, sizeof(alm)); + if (ret) { + dev_err(dev, "Failed to read alarm regs\n"); + return ret; + } + + rtc2tm(&alm.alm0, &a->time); + a->time.tm_mday = -1; + a->time.tm_mon = -1; + a->time.tm_year = -1; + a->enabled = !!(alm.alm_mask & BD70528_MASK_ALM_EN); + a->pending = 0; + + return 0; +} + static int bd70528_read_alarm(struct device *dev, struct rtc_wkalrm *a) { struct bd70528_rtc_alm alm; int ret; struct bd70528_rtc *r = dev_get_drvdata(dev); - struct rohm_regmap_dev *bd70528 = r->mfd; + struct rohm_regmap_dev *parent = r->parent; - ret = regmap_bulk_read(bd70528->regmap, BD70528_REG_RTC_ALM_START, + ret = regmap_bulk_read(parent->regmap, BD70528_REG_RTC_ALM_START, &alm, sizeof(alm)); if (ret) { dev_err(dev, "Failed to read alarm regs\n"); @@ -290,14 +360,14 @@ static int bd70528_set_time_locked(struct device *dev, struct rtc_time *t) int ret, tmpret, old_states; struct bd70528_rtc_data rtc_data; struct bd70528_rtc *r = dev_get_drvdata(dev); - struct rohm_regmap_dev *bd70528 = r->mfd; + struct rohm_regmap_dev *parent = r->parent; ret = bd70528_disable_rtc_based_timers(r, &old_states); if (ret) return ret; - tmpret = regmap_bulk_read(bd70528->regmap, - BD70528_REG_RTC_START, &rtc_data, + tmpret = regmap_bulk_read(parent->regmap, + r->reg_time_start, &rtc_data, sizeof(rtc_data)); if (tmpret) { dev_err(dev, "Failed to read RTC time registers\n"); @@ -305,8 +375,8 @@ static int bd70528_set_time_locked(struct device *dev, struct rtc_time *t) } tm2rtc(t, &rtc_data); - tmpret = regmap_bulk_write(bd70528->regmap, - BD70528_REG_RTC_START, &rtc_data, + tmpret = regmap_bulk_write(parent->regmap, + r->reg_time_start, &rtc_data, sizeof(rtc_data)); if (tmpret) { dev_err(dev, "Failed to set RTC time\n"); @@ -321,27 +391,32 @@ renable_out: return ret; } +static int bd71828_set_time(struct device *dev, struct rtc_time *t) +{ + return bd70528_set_time_locked(dev, t); +} + static int bd70528_set_time(struct device *dev, struct rtc_time *t) { int ret; struct bd70528_rtc *r = dev_get_drvdata(dev); - bd70528_wdt_lock(r->mfd); + bd70528_wdt_lock(r->parent); ret = bd70528_set_time_locked(dev, t); - bd70528_wdt_unlock(r->mfd); + bd70528_wdt_unlock(r->parent); return ret; } static int bd70528_get_time(struct device *dev, struct rtc_time *t) { struct bd70528_rtc *r = dev_get_drvdata(dev); - struct rohm_regmap_dev *bd70528 = r->mfd; + struct rohm_regmap_dev *parent = r->parent; struct bd70528_rtc_data rtc_data; int ret; /* read the RTC date and time registers all at once */ - ret = regmap_bulk_read(bd70528->regmap, - BD70528_REG_RTC_START, &rtc_data, + ret = regmap_bulk_read(parent->regmap, + r->reg_time_start, &rtc_data, sizeof(rtc_data)); if (ret) { dev_err(dev, "Failed to read RTC time (err %d)\n", ret); @@ -362,19 +437,36 @@ static int bd70528_alm_enable(struct device *dev, unsigned int enabled) if (enabled) enableval = 0; - bd70528_wdt_lock(r->mfd); - ret = bd70528_set_wake(r->mfd, enabled, NULL); + bd70528_wdt_lock(r->parent); + ret = bd70528_set_wake(r->parent, enabled, NULL); if (ret) { dev_err(dev, "Failed to change wake state\n"); goto out_unlock; } - ret = regmap_update_bits(r->mfd->regmap, BD70528_REG_RTC_ALM_MASK, + ret = regmap_update_bits(r->parent->regmap, BD70528_REG_RTC_ALM_MASK, BD70528_MASK_ALM_EN, enableval); if (ret) dev_err(dev, "Failed to change alarm state\n"); out_unlock: - bd70528_wdt_unlock(r->mfd); + bd70528_wdt_unlock(r->parent); + return ret; +} + +static int bd71828_alm_enable(struct device *dev, unsigned int enabled) +{ + int ret; + struct bd70528_rtc *r = dev_get_drvdata(dev); + unsigned int enableval = BD70528_MASK_ALM_EN; + + if (!enabled) + enableval = 0; + + ret = regmap_update_bits(r->parent->regmap, BD71828_REG_RTC_ALM0_MASK, + BD70528_MASK_ALM_EN, enableval); + if (ret) + dev_err(dev, "Failed to change alarm state\n"); + return ret; } @@ -386,6 +478,14 @@ static const struct rtc_class_ops bd70528_rtc_ops = { .alarm_irq_enable = bd70528_alm_enable, }; +static const struct rtc_class_ops bd71828_rtc_ops = { + .read_time = bd70528_get_time, + .set_time = bd71828_set_time, + .read_alarm = bd71828_read_alarm, + .set_alarm = bd71828_set_alarm, + .alarm_irq_enable = bd71828_alm_enable, +}; + static irqreturn_t alm_hndlr(int irq, void *data) { struct rtc_device *rtc = data; @@ -397,14 +497,19 @@ static irqreturn_t alm_hndlr(int irq, void *data) static int bd70528_probe(struct platform_device *pdev) { struct bd70528_rtc *bd_rtc; - struct rohm_regmap_dev *mfd; + const struct rtc_class_ops *rtc_ops; + struct rohm_regmap_dev *parent; + const char *irq_name; int ret; struct rtc_device *rtc; int irq; unsigned int hr; + bool enable_main_irq = false; + u8 hour_reg; + enum rohm_chip_type chip = platform_get_device_id(pdev)->driver_data; - mfd = dev_get_drvdata(pdev->dev.parent); - if (!mfd) { + parent = dev_get_drvdata(pdev->dev.parent); + if (!parent) { dev_err(&pdev->dev, "No MFD driver data\n"); return -EINVAL; } @@ -412,16 +517,39 @@ static int bd70528_probe(struct platform_device *pdev) if (!bd_rtc) return -ENOMEM; - bd_rtc->mfd = mfd; + bd_rtc->parent = parent; bd_rtc->dev = &pdev->dev; - irq = platform_get_irq_byname(pdev, "bd70528-rtc-alm"); - if (irq < 0) + switch (chip) { + case ROHM_CHIP_TYPE_BD70528: + irq_name = "bd70528-rtc-alm"; + bd_rtc->has_rtc_timers = true; + bd_rtc->reg_time_start = BD70528_REG_RTC_START; + hour_reg = BD70528_REG_RTC_HOUR; + enable_main_irq = true; + rtc_ops = &bd70528_rtc_ops; + break; + case ROHM_CHIP_TYPE_BD71828: + irq_name = "bd71828-rtc-alm-0"; + bd_rtc->reg_time_start = BD71828_REG_RTC_START; + hour_reg = BD71828_REG_RTC_HOUR; + rtc_ops = &bd71828_rtc_ops; + break; + default: + dev_err(&pdev->dev, "Unknown chip\n"); + return -ENOENT; + } + + irq = platform_get_irq_byname(pdev, irq_name); + + if (irq < 0) { + dev_err(&pdev->dev, "Failed to get irq\n"); return irq; + } platform_set_drvdata(pdev, bd_rtc); - ret = regmap_read(mfd->regmap, BD70528_REG_RTC_HOUR, &hr); + ret = regmap_read(parent->regmap, hour_reg, &hr); if (ret) { dev_err(&pdev->dev, "Failed to reag RTC clock\n"); @@ -431,10 +559,10 @@ static int bd70528_probe(struct platform_device *pdev) if (!(hr & BD70528_MASK_RTC_HOUR_24H)) { struct rtc_time t; - ret = bd70528_get_time(&pdev->dev, &t); + ret = rtc_ops->read_time(&pdev->dev, &t); if (!ret) - ret = bd70528_set_time(&pdev->dev, &t); + ret = rtc_ops->set_time(&pdev->dev, &t); if (ret) { dev_err(&pdev->dev, @@ -454,7 +582,7 @@ static int bd70528_probe(struct platform_device *pdev) rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; rtc->range_max = RTC_TIMESTAMP_END_2099; - rtc->ops = &bd70528_rtc_ops; + rtc->ops = rtc_ops; /* Request alarm IRQ prior to registerig the RTC */ ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, &alm_hndlr, @@ -468,27 +596,37 @@ static int bd70528_probe(struct platform_device *pdev) * leave them enabled as irq-controller should disable irqs * from sub-registers when IRQ is disabled or freed. */ - ret = regmap_update_bits(mfd->regmap, + if (enable_main_irq) { + ret = regmap_update_bits(parent->regmap, BD70528_REG_INT_MAIN_MASK, BD70528_INT_RTC_MASK, 0); - if (ret) { - dev_err(&pdev->dev, "Failed to enable RTC interrupts\n"); - return ret; + if (ret) { + dev_err(&pdev->dev, "Failed to enable RTC interrupts\n"); + return ret; + } } return rtc_register_device(rtc); } +static const struct platform_device_id bd718x7_rtc_id[] = { + { "bd70528-rtc", ROHM_CHIP_TYPE_BD70528 }, + { "bd71828-rtc", ROHM_CHIP_TYPE_BD71828 }, + { }, +}; +MODULE_DEVICE_TABLE(platform, bd718x7_rtc_id); + static struct platform_driver bd70528_rtc = { .driver = { .name = "bd70528-rtc" }, .probe = bd70528_probe, + .id_table = bd718x7_rtc_id, }; module_platform_driver(bd70528_rtc); MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>"); -MODULE_DESCRIPTION("BD70528 RTC driver"); +MODULE_DESCRIPTION("ROHM BD70528 and BD71828 PMIC RTC driver"); MODULE_LICENSE("GPL"); MODULE_ALIAS("platform:bd70528-rtc"); diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index 033303708c8b..b795fe4cbd2e 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -850,7 +850,7 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq) rtc_cmos_int_handler = cmos_interrupt; retval = request_irq(rtc_irq, rtc_cmos_int_handler, - IRQF_SHARED, dev_name(&cmos_rtc.rtc->dev), + 0, dev_name(&cmos_rtc.rtc->dev), cmos_rtc.rtc); if (retval < 0) { dev_dbg(dev, "IRQ %d is already in use\n", rtc_irq); @@ -1197,8 +1197,6 @@ static void rtc_wake_off(struct device *dev) /* Enable use_acpi_alarm mode for Intel platforms no earlier than 2015 */ static void use_acpi_alarm_quirks(void) { - int year; - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return; @@ -1208,8 +1206,10 @@ static void use_acpi_alarm_quirks(void) if (!is_hpet_enabled()) return; - if (dmi_get_date(DMI_BIOS_DATE, &year, NULL, NULL) && year >= 2015) - use_acpi_alarm = true; + if (dmi_get_bios_year() < 2015) + return; + + use_acpi_alarm = true; } #else static inline void use_acpi_alarm_quirks(void) { } @@ -1305,7 +1305,7 @@ static int cmos_pnp_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) * hardcode it on systems with a legacy PIC. */ if (nr_legacy_irqs()) - irq = 8; + irq = RTC_IRQ; #endif return cmos_do_probe(&pnp->dev, pnp_get_resource(pnp, IORESOURCE_IO, 0), irq); diff --git a/drivers/rtc/rtc-cros-ec.c b/drivers/rtc/rtc-cros-ec.c index d043d30f05bc..f7343c289cab 100644 --- a/drivers/rtc/rtc-cros-ec.c +++ b/drivers/rtc/rtc-cros-ec.c @@ -5,7 +5,6 @@ // Author: Stephen Barber <smbarber@chromium.org> #include <linux/kernel.h> -#include <linux/mfd/cros_ec.h> #include <linux/module.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> diff --git a/drivers/rtc/rtc-ds1343.c b/drivers/rtc/rtc-ds1343.c index d21004a68ee0..ba143423875b 100644 --- a/drivers/rtc/rtc-ds1343.c +++ b/drivers/rtc/rtc-ds1343.c @@ -75,7 +75,6 @@ static const struct spi_device_id ds1343_id[] = { MODULE_DEVICE_TABLE(spi, ds1343_id); struct ds1343_priv { - struct spi_device *spi; struct rtc_device *rtc; struct regmap *map; int irq; @@ -362,12 +361,13 @@ static int ds1343_probe(struct spi_device *spi) if (!priv) return -ENOMEM; - priv->spi = spi; - /* RTC DS1347 works in spi mode 3 and - * its chip select is active high + * its chip select is active high. Active high should be defined as + * "inverse polarity" as GPIO-based chip selects can be logically + * active high but inverted by the GPIO library. */ - spi->mode = SPI_MODE_3 | SPI_CS_HIGH; + spi->mode |= SPI_MODE_3; + spi->mode ^= SPI_CS_HIGH; spi->bits_per_word = 8; res = spi_setup(spi); if (res) diff --git a/drivers/rtc/rtc-hym8563.c b/drivers/rtc/rtc-hym8563.c index 443f6d05ce29..0fb79c4afb46 100644 --- a/drivers/rtc/rtc-hym8563.c +++ b/drivers/rtc/rtc-hym8563.c @@ -78,7 +78,6 @@ struct hym8563 { struct i2c_client *client; struct rtc_device *rtc; - bool valid; #ifdef CONFIG_COMMON_CLK struct clk_hw clkout_hw; #endif @@ -91,19 +90,19 @@ struct hym8563 { static int hym8563_rtc_read_time(struct device *dev, struct rtc_time *tm) { struct i2c_client *client = to_i2c_client(dev); - struct hym8563 *hym8563 = i2c_get_clientdata(client); u8 buf[7]; int ret; - if (!hym8563->valid) { - dev_warn(&client->dev, "no valid clock/calendar values available\n"); - return -EPERM; - } - ret = i2c_smbus_read_i2c_block_data(client, HYM8563_SEC, 7, buf); if (ret < 0) return ret; + if (buf[0] & HYM8563_SEC_VL) { + dev_warn(&client->dev, + "no valid clock/calendar values available\n"); + return -EINVAL; + } + tm->tm_sec = bcd2bin(buf[0] & HYM8563_SEC_MASK); tm->tm_min = bcd2bin(buf[1] & HYM8563_MIN_MASK); tm->tm_hour = bcd2bin(buf[2] & HYM8563_HOUR_MASK); @@ -118,7 +117,6 @@ static int hym8563_rtc_read_time(struct device *dev, struct rtc_time *tm) static int hym8563_rtc_set_time(struct device *dev, struct rtc_time *tm) { struct i2c_client *client = to_i2c_client(dev); - struct hym8563 *hym8563 = i2c_get_clientdata(client); u8 buf[7]; int ret; @@ -157,8 +155,6 @@ static int hym8563_rtc_set_time(struct device *dev, struct rtc_time *tm) if (ret < 0) return ret; - hym8563->valid = true; - return 0; } @@ -556,9 +552,8 @@ static int hym8563_probe(struct i2c_client *client, if (ret < 0) return ret; - hym8563->valid = !(ret & HYM8563_SEC_VL); dev_dbg(&client->dev, "rtc information is %s\n", - hym8563->valid ? "valid" : "invalid"); + (ret & HYM8563_SEC_VL) ? "invalid" : "valid"); hym8563->rtc = devm_rtc_device_register(&client->dev, client->name, &hym8563_rtc_ops, THIS_MODULE); diff --git a/drivers/rtc/rtc-moxart.c b/drivers/rtc/rtc-moxart.c index 07b30a373a92..6b24ac9e1cfa 100644 --- a/drivers/rtc/rtc-moxart.c +++ b/drivers/rtc/rtc-moxart.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * MOXA ART RTC driver. * @@ -7,10 +8,6 @@ * * Based on code from * Moxa Technology Co., Ltd. <www.moxa.com> - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #include <linux/init.h> diff --git a/drivers/rtc/rtc-mt6397.c b/drivers/rtc/rtc-mt6397.c index 9135e2101752..cda238dfe69b 100644 --- a/drivers/rtc/rtc-mt6397.c +++ b/drivers/rtc/rtc-mt6397.c @@ -297,15 +297,7 @@ static int mtk_rtc_probe(struct platform_device *pdev) rtc->rtc_dev->ops = &mtk_rtc_ops; - ret = rtc_register_device(rtc->rtc_dev); - if (ret) - goto out_free_irq; - - return 0; - -out_free_irq: - free_irq(rtc->irq, rtc); - return ret; + return rtc_register_device(rtc->rtc_dev); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/rtc/rtc-omap.c b/drivers/rtc/rtc-omap.c index 988a4dfcfaf8..d4ed20fb3194 100644 --- a/drivers/rtc/rtc-omap.c +++ b/drivers/rtc/rtc-omap.c @@ -616,7 +616,7 @@ static int rtc_pinconf_get(struct pinctrl_dev *pctldev, break; default: return -ENOTSUPP; - }; + } *config = pinconf_to_config_packed(param, arg); diff --git a/drivers/rtc/rtc-pcf2127.c b/drivers/rtc/rtc-pcf2127.c index ba5baaca47be..4e50d6768f13 100644 --- a/drivers/rtc/rtc-pcf2127.c +++ b/drivers/rtc/rtc-pcf2127.c @@ -199,11 +199,9 @@ static int pcf2127_rtc_ioctl(struct device *dev, if (ret) return ret; - touser = touser & PCF2127_BIT_CTRL3_BLF ? 1 : 0; + touser = touser & PCF2127_BIT_CTRL3_BLF ? RTC_VL_BACKUP_LOW : 0; - if (copy_to_user((void __user *)arg, &touser, sizeof(int))) - return -EFAULT; - return 0; + return put_user(touser, (unsigned int __user *)arg); default: return -ENOIOCTLCMD; } diff --git a/drivers/rtc/rtc-pcf85063.c b/drivers/rtc/rtc-pcf85063.c index 1afa6d9fa9fb..1db17ba1fc64 100644 --- a/drivers/rtc/rtc-pcf85063.c +++ b/drivers/rtc/rtc-pcf85063.c @@ -289,21 +289,9 @@ static int pcf85063_ioctl(struct device *dev, unsigned int cmd, if (ret < 0) return ret; - if (status & PCF85063_REG_SC_OS) - dev_warn(&pcf85063->rtc->dev, "Voltage low, data loss detected.\n"); + status = status & PCF85063_REG_SC_OS ? RTC_VL_DATA_INVALID : 0; - status &= PCF85063_REG_SC_OS; - - if (copy_to_user((void __user *)arg, &status, sizeof(int))) - return -EFAULT; - - return 0; - - case RTC_VL_CLR: - ret = regmap_update_bits(pcf85063->regmap, PCF85063_REG_SC, - PCF85063_REG_SC_OS, 0); - - return ret; + return put_user(status, (unsigned int __user *)arg); default: return -ENOIOCTLCMD; diff --git a/drivers/rtc/rtc-pcf8523.c b/drivers/rtc/rtc-pcf8523.c index b24c908f5f06..47e0f411dd5c 100644 --- a/drivers/rtc/rtc-pcf8523.c +++ b/drivers/rtc/rtc-pcf8523.c @@ -282,11 +282,11 @@ static int pcf8523_rtc_ioctl(struct device *dev, unsigned int cmd, ret = pcf8523_voltage_low(client); if (ret < 0) return ret; + if (ret) + ret = RTC_VL_BACKUP_LOW; - if (copy_to_user((void __user *)arg, &ret, sizeof(int))) - return -EFAULT; + return put_user(ret, (unsigned int __user *)arg); - return 0; default: return -ENOIOCTLCMD; } diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index 3c322f3079b0..2dc30eafa639 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -22,8 +22,8 @@ #define PCF8563_REG_ST1 0x00 /* status */ #define PCF8563_REG_ST2 0x01 -#define PCF8563_BIT_AIE (1 << 1) -#define PCF8563_BIT_AF (1 << 3) +#define PCF8563_BIT_AIE BIT(1) +#define PCF8563_BIT_AF BIT(3) #define PCF8563_BITS_ST2_N (7 << 5) #define PCF8563_REG_SC 0x02 /* datetime */ @@ -76,7 +76,6 @@ struct pcf8563 { * 1970...2069. */ int c_polarity; /* 0: MO_C=1 means 19xx, otherwise MO_C=1 means 20xx */ - int voltage_low; /* incicates if a low_voltage was detected */ struct i2c_client *client; #ifdef CONFIG_COMMON_CLK @@ -208,7 +207,6 @@ static int pcf8563_rtc_read_time(struct device *dev, struct rtc_time *tm) return err; if (buf[PCF8563_REG_SC] & PCF8563_SC_LV) { - pcf8563->voltage_low = 1; dev_err(&client->dev, "low voltage detected, date/time is not reliable.\n"); return -EINVAL; @@ -276,43 +274,23 @@ static int pcf8563_rtc_set_time(struct device *dev, struct rtc_time *tm) 9 - PCF8563_REG_SC, buf + PCF8563_REG_SC); } -#ifdef CONFIG_RTC_INTF_DEV static int pcf8563_rtc_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) { - struct pcf8563 *pcf8563 = i2c_get_clientdata(to_i2c_client(dev)); - struct rtc_time tm; + struct i2c_client *client = to_i2c_client(dev); + int ret; switch (cmd) { case RTC_VL_READ: - if (pcf8563->voltage_low) - dev_info(dev, "low voltage detected, date/time is not reliable.\n"); - - if (copy_to_user((void __user *)arg, &pcf8563->voltage_low, - sizeof(int))) - return -EFAULT; - return 0; - case RTC_VL_CLR: - /* - * Clear the VL bit in the seconds register in case - * the time has not been set already (which would - * have cleared it). This does not really matter - * because of the cached voltage_low value but do it - * anyway for consistency. - */ - if (pcf8563_rtc_read_time(dev, &tm)) - pcf8563_rtc_set_time(dev, &tm); - - /* Clear the cached value. */ - pcf8563->voltage_low = 0; + ret = i2c_smbus_read_byte_data(client, PCF8563_REG_SC); + if (ret < 0) + return ret; - return 0; + return put_user(ret & PCF8563_SC_LV ? RTC_VL_DATA_INVALID : 0, + (unsigned int __user *)arg); default: return -ENOIOCTLCMD; } } -#else -#define pcf8563_rtc_ioctl NULL -#endif static int pcf8563_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *tm) { diff --git a/drivers/rtc/rtc-rv3028.c b/drivers/rtc/rtc-rv3028.c index 6b7b3a69601a..a0ddc86c975a 100644 --- a/drivers/rtc/rtc-rv3028.c +++ b/drivers/rtc/rtc-rv3028.c @@ -428,21 +428,8 @@ static int rv3028_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) if (ret < 0) return ret; - if (status & RV3028_STATUS_PORF) - dev_warn(&rv3028->rtc->dev, "Voltage low, data loss detected.\n"); - - status &= RV3028_STATUS_PORF; - - if (copy_to_user((void __user *)arg, &status, sizeof(int))) - return -EFAULT; - - return 0; - - case RTC_VL_CLR: - ret = regmap_update_bits(rv3028->regmap, RV3028_STATUS, - RV3028_STATUS_PORF, 0); - - return ret; + status = status & RV3028_STATUS_PORF ? RTC_VL_DATA_INVALID : 0; + return put_user(status, (unsigned int __user *)arg); default: return -ENOIOCTLCMD; diff --git a/drivers/rtc/rtc-rv3029c2.c b/drivers/rtc/rtc-rv3029c2.c index 4cdf6588e1d9..62718231731b 100644 --- a/drivers/rtc/rtc-rv3029c2.c +++ b/drivers/rtc/rtc-rv3029c2.c @@ -109,10 +109,8 @@ #define RV3029_CONTROL_E2P_TOV_MASK 0x3F /* XTAL turnover temp mask */ /* user ram section */ -#define RV3029_USR1_RAM_PAGE 0x38 -#define RV3029_USR1_SECTION_LEN 0x04 -#define RV3029_USR2_RAM_PAGE 0x3C -#define RV3029_USR2_SECTION_LEN 0x04 +#define RV3029_RAM_PAGE 0x38 +#define RV3029_RAM_SECTION_LEN 8 struct rv3029_data { struct device *dev; @@ -121,77 +119,13 @@ struct rv3029_data { int irq; }; -static int rv3029_read_regs(struct device *dev, u8 reg, u8 *buf, - unsigned int len) -{ - struct rv3029_data *rv3029 = dev_get_drvdata(dev); - - if ((reg > RV3029_USR1_RAM_PAGE + 7) || - (reg + len > RV3029_USR1_RAM_PAGE + 8)) - return -EINVAL; - - return regmap_bulk_read(rv3029->regmap, reg, buf, len); -} - -static int rv3029_write_regs(struct device *dev, u8 reg, u8 const buf[], - unsigned int len) -{ - struct rv3029_data *rv3029 = dev_get_drvdata(dev); - - if ((reg > RV3029_USR1_RAM_PAGE + 7) || - (reg + len > RV3029_USR1_RAM_PAGE + 8)) - return -EINVAL; - - return regmap_bulk_write(rv3029->regmap, reg, buf, len); -} - -static int rv3029_update_bits(struct device *dev, u8 reg, u8 mask, u8 set) -{ - u8 buf; - int ret; - - ret = rv3029_read_regs(dev, reg, &buf, 1); - if (ret < 0) - return ret; - buf &= ~mask; - buf |= set & mask; - ret = rv3029_write_regs(dev, reg, &buf, 1); - if (ret < 0) - return ret; - - return 0; -} - -static int rv3029_get_sr(struct device *dev, u8 *buf) -{ - int ret = rv3029_read_regs(dev, RV3029_STATUS, buf, 1); - - if (ret < 0) - return -EIO; - dev_dbg(dev, "status = 0x%.2x (%d)\n", buf[0], buf[0]); - return 0; -} - -static int rv3029_set_sr(struct device *dev, u8 val) -{ - u8 buf[1]; - int sr; - - buf[0] = val; - sr = rv3029_write_regs(dev, RV3029_STATUS, buf, 1); - dev_dbg(dev, "status = 0x%.2x (%d)\n", buf[0], buf[0]); - if (sr < 0) - return -EIO; - return 0; -} - -static int rv3029_eeprom_busywait(struct device *dev) +static int rv3029_eeprom_busywait(struct rv3029_data *rv3029) { + unsigned int sr; int i, ret; - u8 sr; for (i = 100; i > 0; i--) { - ret = rv3029_get_sr(dev, &sr); + ret = regmap_read(rv3029->regmap, RV3029_STATUS, &sr); if (ret < 0) break; if (!(sr & RV3029_STATUS_EEBUSY)) @@ -199,126 +133,128 @@ static int rv3029_eeprom_busywait(struct device *dev) usleep_range(1000, 10000); } if (i <= 0) { - dev_err(dev, "EEPROM busy wait timeout.\n"); + dev_err(rv3029->dev, "EEPROM busy wait timeout.\n"); return -ETIMEDOUT; } return ret; } -static int rv3029_eeprom_exit(struct device *dev) +static int rv3029_eeprom_exit(struct rv3029_data *rv3029) { /* Re-enable eeprom refresh */ - return rv3029_update_bits(dev, RV3029_ONOFF_CTRL, + return regmap_update_bits(rv3029->regmap, RV3029_ONOFF_CTRL, RV3029_ONOFF_CTRL_EERE, RV3029_ONOFF_CTRL_EERE); } -static int rv3029_eeprom_enter(struct device *dev) +static int rv3029_eeprom_enter(struct rv3029_data *rv3029) { + unsigned int sr; int ret; - u8 sr; /* Check whether we are in the allowed voltage range. */ - ret = rv3029_get_sr(dev, &sr); + ret = regmap_read(rv3029->regmap, RV3029_STATUS, &sr); if (ret < 0) return ret; - if (sr & (RV3029_STATUS_VLOW1 | RV3029_STATUS_VLOW2)) { + if (sr & RV3029_STATUS_VLOW2) + return -ENODEV; + if (sr & RV3029_STATUS_VLOW1) { /* We clear the bits and retry once just in case * we had a brown out in early startup. */ - sr &= ~RV3029_STATUS_VLOW1; - sr &= ~RV3029_STATUS_VLOW2; - ret = rv3029_set_sr(dev, sr); + ret = regmap_update_bits(rv3029->regmap, RV3029_STATUS, + RV3029_STATUS_VLOW1, 0); if (ret < 0) return ret; usleep_range(1000, 10000); - ret = rv3029_get_sr(dev, &sr); + ret = regmap_read(rv3029->regmap, RV3029_STATUS, &sr); if (ret < 0) return ret; - if (sr & (RV3029_STATUS_VLOW1 | RV3029_STATUS_VLOW2)) { - dev_err(dev, + if (sr & RV3029_STATUS_VLOW1) { + dev_err(rv3029->dev, "Supply voltage is too low to safely access the EEPROM.\n"); return -ENODEV; } } /* Disable eeprom refresh. */ - ret = rv3029_update_bits(dev, RV3029_ONOFF_CTRL, RV3029_ONOFF_CTRL_EERE, - 0); + ret = regmap_update_bits(rv3029->regmap, RV3029_ONOFF_CTRL, + RV3029_ONOFF_CTRL_EERE, 0); if (ret < 0) return ret; /* Wait for any previous eeprom accesses to finish. */ - ret = rv3029_eeprom_busywait(dev); + ret = rv3029_eeprom_busywait(rv3029); if (ret < 0) - rv3029_eeprom_exit(dev); + rv3029_eeprom_exit(rv3029); return ret; } -static int rv3029_eeprom_read(struct device *dev, u8 reg, +static int rv3029_eeprom_read(struct rv3029_data *rv3029, u8 reg, u8 buf[], size_t len) { int ret, err; - err = rv3029_eeprom_enter(dev); + err = rv3029_eeprom_enter(rv3029); if (err < 0) return err; - ret = rv3029_read_regs(dev, reg, buf, len); + ret = regmap_bulk_read(rv3029->regmap, reg, buf, len); - err = rv3029_eeprom_exit(dev); + err = rv3029_eeprom_exit(rv3029); if (err < 0) return err; return ret; } -static int rv3029_eeprom_write(struct device *dev, u8 reg, +static int rv3029_eeprom_write(struct rv3029_data *rv3029, u8 reg, u8 const buf[], size_t len) { + unsigned int tmp; int ret, err; size_t i; - u8 tmp; - err = rv3029_eeprom_enter(dev); + err = rv3029_eeprom_enter(rv3029); if (err < 0) return err; for (i = 0; i < len; i++, reg++) { - ret = rv3029_read_regs(dev, reg, &tmp, 1); + ret = regmap_read(rv3029->regmap, reg, &tmp); if (ret < 0) break; if (tmp != buf[i]) { - ret = rv3029_write_regs(dev, reg, &buf[i], 1); + tmp = buf[i]; + ret = regmap_write(rv3029->regmap, reg, tmp); if (ret < 0) break; } - ret = rv3029_eeprom_busywait(dev); + ret = rv3029_eeprom_busywait(rv3029); if (ret < 0) break; } - err = rv3029_eeprom_exit(dev); + err = rv3029_eeprom_exit(rv3029); if (err < 0) return err; return ret; } -static int rv3029_eeprom_update_bits(struct device *dev, +static int rv3029_eeprom_update_bits(struct rv3029_data *rv3029, u8 reg, u8 mask, u8 set) { u8 buf; int ret; - ret = rv3029_eeprom_read(dev, reg, &buf, 1); + ret = rv3029_eeprom_read(rv3029, reg, &buf, 1); if (ret < 0) return ret; buf &= ~mask; buf |= set & mask; - ret = rv3029_eeprom_write(dev, reg, &buf, 1); + ret = rv3029_eeprom_write(rv3029, reg, &buf, 1); if (ret < 0) return ret; @@ -330,20 +266,20 @@ static irqreturn_t rv3029_handle_irq(int irq, void *dev_id) struct device *dev = dev_id; struct rv3029_data *rv3029 = dev_get_drvdata(dev); struct mutex *lock = &rv3029->rtc->ops_lock; + unsigned int flags, controls; unsigned long events = 0; - u8 flags, controls; int ret; mutex_lock(lock); - ret = rv3029_read_regs(dev, RV3029_IRQ_CTRL, &controls, 1); + ret = regmap_read(rv3029->regmap, RV3029_IRQ_CTRL, &controls); if (ret) { dev_warn(dev, "Read IRQ Control Register error %d\n", ret); mutex_unlock(lock); return IRQ_NONE; } - ret = rv3029_read_regs(dev, RV3029_IRQ_FLAGS, &flags, 1); + ret = regmap_read(rv3029->regmap, RV3029_IRQ_FLAGS, &flags); if (ret) { dev_warn(dev, "Read IRQ Flags Register error %d\n", ret); mutex_unlock(lock); @@ -358,8 +294,8 @@ static irqreturn_t rv3029_handle_irq(int irq, void *dev_id) if (events) { rtc_update_irq(rv3029->rtc, 1, events); - rv3029_write_regs(dev, RV3029_IRQ_FLAGS, &flags, 1); - rv3029_write_regs(dev, RV3029_IRQ_CTRL, &controls, 1); + regmap_write(rv3029->regmap, RV3029_IRQ_FLAGS, flags); + regmap_write(rv3029->regmap, RV3029_IRQ_CTRL, controls); } mutex_unlock(lock); @@ -368,22 +304,22 @@ static irqreturn_t rv3029_handle_irq(int irq, void *dev_id) static int rv3029_read_time(struct device *dev, struct rtc_time *tm) { - u8 buf[1]; + struct rv3029_data *rv3029 = dev_get_drvdata(dev); + unsigned int sr; int ret; u8 regs[RV3029_WATCH_SECTION_LEN] = { 0, }; - ret = rv3029_get_sr(dev, buf); - if (ret < 0) { - dev_err(dev, "%s: reading SR failed\n", __func__); - return -EIO; - } + ret = regmap_read(rv3029->regmap, RV3029_STATUS, &sr); + if (ret < 0) + return ret; + + if (sr & (RV3029_STATUS_VLOW2 | RV3029_STATUS_PON)) + return -EINVAL; - ret = rv3029_read_regs(dev, RV3029_W_SEC, regs, + ret = regmap_bulk_read(rv3029->regmap, RV3029_W_SEC, regs, RV3029_WATCH_SECTION_LEN); - if (ret < 0) { - dev_err(dev, "%s: reading RTC section failed\n", __func__); + if (ret < 0) return ret; - } tm->tm_sec = bcd2bin(regs[RV3029_W_SEC - RV3029_W_SEC]); tm->tm_min = bcd2bin(regs[RV3029_W_MINUTES - RV3029_W_SEC]); @@ -411,34 +347,24 @@ static int rv3029_read_time(struct device *dev, struct rtc_time *tm) static int rv3029_read_alarm(struct device *dev, struct rtc_wkalrm *alarm) { + struct rv3029_data *rv3029 = dev_get_drvdata(dev); struct rtc_time *const tm = &alarm->time; + unsigned int controls, flags; int ret; - u8 regs[8], controls, flags; - - ret = rv3029_get_sr(dev, regs); - if (ret < 0) { - dev_err(dev, "%s: reading SR failed\n", __func__); - return -EIO; - } + u8 regs[8]; - ret = rv3029_read_regs(dev, RV3029_A_SC, regs, + ret = regmap_bulk_read(rv3029->regmap, RV3029_A_SC, regs, RV3029_ALARM_SECTION_LEN); - - if (ret < 0) { - dev_err(dev, "%s: reading alarm section failed\n", __func__); + if (ret < 0) return ret; - } - ret = rv3029_read_regs(dev, RV3029_IRQ_CTRL, &controls, 1); - if (ret) { - dev_err(dev, "Read IRQ Control Register error %d\n", ret); + ret = regmap_read(rv3029->regmap, RV3029_IRQ_CTRL, &controls); + if (ret) return ret; - } - ret = rv3029_read_regs(dev, RV3029_IRQ_FLAGS, &flags, 1); - if (ret < 0) { - dev_err(dev, "Read IRQ Flags Register error %d\n", ret); + + ret = regmap_read(rv3029->regmap, RV3029_IRQ_FLAGS, &flags); + if (ret < 0) return ret; - } tm->tm_sec = bcd2bin(regs[RV3029_A_SC - RV3029_A_SC] & 0x7f); tm->tm_min = bcd2bin(regs[RV3029_A_MN - RV3029_A_SC] & 0x7f); @@ -456,50 +382,20 @@ static int rv3029_read_alarm(struct device *dev, struct rtc_wkalrm *alarm) static int rv3029_alarm_irq_enable(struct device *dev, unsigned int enable) { - int ret; - u8 controls; - - ret = rv3029_read_regs(dev, RV3029_IRQ_CTRL, &controls, 1); - if (ret < 0) { - dev_warn(dev, "Read IRQ Control Register error %d\n", ret); - return ret; - } - - /* enable/disable AIE irq */ - if (enable) - controls |= RV3029_IRQ_CTRL_AIE; - else - controls &= ~RV3029_IRQ_CTRL_AIE; - - ret = rv3029_write_regs(dev, RV3029_IRQ_CTRL, &controls, 1); - if (ret < 0) { - dev_err(dev, "can't update INT reg\n"); - return ret; - } + struct rv3029_data *rv3029 = dev_get_drvdata(dev); - return 0; + return regmap_update_bits(rv3029->regmap, RV3029_IRQ_CTRL, + RV3029_IRQ_CTRL_AIE, + enable ? RV3029_IRQ_CTRL_AIE : 0); } static int rv3029_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) { + struct rv3029_data *rv3029 = dev_get_drvdata(dev); struct rtc_time *const tm = &alarm->time; int ret; u8 regs[8]; - /* - * The clock has an 8 bit wide bcd-coded register (they never learn) - * for the year. tm_year is an offset from 1900 and we are interested - * in the 2000-2099 range, so any value less than 100 is invalid. - */ - if (tm->tm_year < 100) - return -EINVAL; - - ret = rv3029_get_sr(dev, regs); - if (ret < 0) { - dev_err(dev, "%s: reading SR failed\n", __func__); - return -EIO; - } - /* Activate all the alarms with AE_x bit */ regs[RV3029_A_SC - RV3029_A_SC] = bin2bcd(tm->tm_sec) | RV3029_A_AE_X; regs[RV3029_A_MN - RV3029_A_SC] = bin2bcd(tm->tm_min) | RV3029_A_AE_X; @@ -515,39 +411,20 @@ static int rv3029_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) | RV3029_A_AE_X; /* Write the alarm */ - ret = rv3029_write_regs(dev, RV3029_A_SC, regs, + ret = regmap_bulk_write(rv3029->regmap, RV3029_A_SC, regs, RV3029_ALARM_SECTION_LEN); if (ret < 0) return ret; - if (alarm->enabled) { - /* enable AIE irq */ - ret = rv3029_alarm_irq_enable(dev, 1); - if (ret) - return ret; - } else { - /* disable AIE irq */ - ret = rv3029_alarm_irq_enable(dev, 0); - if (ret) - return ret; - } - - return 0; + return rv3029_alarm_irq_enable(dev, alarm->enabled); } static int rv3029_set_time(struct device *dev, struct rtc_time *tm) { + struct rv3029_data *rv3029 = dev_get_drvdata(dev); u8 regs[8]; int ret; - /* - * The clock has an 8 bit wide bcd-coded register (they never learn) - * for the year. tm_year is an offset from 1900 and we are interested - * in the 2000-2099 range, so any value less than 100 is invalid. - */ - if (tm->tm_year < 100) - return -EINVAL; - regs[RV3029_W_SEC - RV3029_W_SEC] = bin2bcd(tm->tm_sec); regs[RV3029_W_MINUTES - RV3029_W_SEC] = bin2bcd(tm->tm_min); regs[RV3029_W_HOURS - RV3029_W_SEC] = bin2bcd(tm->tm_hour); @@ -556,24 +433,55 @@ static int rv3029_set_time(struct device *dev, struct rtc_time *tm) regs[RV3029_W_DAYS - RV3029_W_SEC] = bin2bcd(tm->tm_wday + 1) & 0x7; regs[RV3029_W_YEARS - RV3029_W_SEC] = bin2bcd(tm->tm_year - 100); - ret = rv3029_write_regs(dev, RV3029_W_SEC, regs, + ret = regmap_bulk_write(rv3029->regmap, RV3029_W_SEC, regs, RV3029_WATCH_SECTION_LEN); if (ret < 0) return ret; - ret = rv3029_get_sr(dev, regs); - if (ret < 0) { - dev_err(dev, "%s: reading SR failed\n", __func__); - return ret; - } - /* clear PON bit */ - ret = rv3029_set_sr(dev, (regs[0] & ~RV3029_STATUS_PON)); - if (ret < 0) { - dev_err(dev, "%s: reading SR failed\n", __func__); - return ret; + /* clear PON and VLOW2 bits */ + return regmap_update_bits(rv3029->regmap, RV3029_STATUS, + RV3029_STATUS_PON | RV3029_STATUS_VLOW2, 0); +} + +static int rv3029_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) +{ + struct rv3029_data *rv3029 = dev_get_drvdata(dev); + unsigned long vl = 0; + int sr, ret = 0; + + switch (cmd) { + case RTC_VL_READ: + ret = regmap_read(rv3029->regmap, RV3029_STATUS, &sr); + if (ret < 0) + return ret; + + if (sr & RV3029_STATUS_VLOW1) + vl = RTC_VL_ACCURACY_LOW; + + if (sr & (RV3029_STATUS_VLOW2 | RV3029_STATUS_PON)) + vl |= RTC_VL_DATA_INVALID; + + return put_user(vl, (unsigned int __user *)arg); + + case RTC_VL_CLR: + return regmap_update_bits(rv3029->regmap, RV3029_STATUS, + RV3029_STATUS_VLOW1, 0); + + default: + return -ENOIOCTLCMD; } +} - return 0; +static int rv3029_nvram_write(void *priv, unsigned int offset, void *val, + size_t bytes) +{ + return regmap_bulk_write(priv, RV3029_RAM_PAGE + offset, val, bytes); +} + +static int rv3029_nvram_read(void *priv, unsigned int offset, void *val, + size_t bytes) +{ + return regmap_bulk_read(priv, RV3029_RAM_PAGE + offset, val, bytes); } static const struct rv3029_trickle_tab_elem { @@ -635,6 +543,7 @@ static const struct rv3029_trickle_tab_elem { static void rv3029_trickle_config(struct device *dev) { + struct rv3029_data *rv3029 = dev_get_drvdata(dev); struct device_node *of_node = dev->of_node; const struct rv3029_trickle_tab_elem *elem; int i, err; @@ -661,7 +570,7 @@ static void rv3029_trickle_config(struct device *dev) "Trickle charger enabled at %d ohms resistance.\n", elem->r); } - err = rv3029_eeprom_update_bits(dev, RV3029_CONTROL_E2P_EECTRL, + err = rv3029_eeprom_update_bits(rv3029, RV3029_CONTROL_E2P_EECTRL, RV3029_TRICKLE_MASK, trickle_set_bits); if (err < 0) @@ -670,12 +579,12 @@ static void rv3029_trickle_config(struct device *dev) #ifdef CONFIG_RTC_DRV_RV3029_HWMON -static int rv3029_read_temp(struct device *dev, int *temp_mC) +static int rv3029_read_temp(struct rv3029_data *rv3029, int *temp_mC) { + unsigned int temp; int ret; - u8 temp; - ret = rv3029_read_regs(dev, RV3029_TEMP_PAGE, &temp, 1); + ret = regmap_read(rv3029->regmap, RV3029_TEMP_PAGE, &temp); if (ret < 0) return ret; @@ -688,9 +597,10 @@ static ssize_t rv3029_hwmon_show_temp(struct device *dev, struct device_attribute *attr, char *buf) { + struct rv3029_data *rv3029 = dev_get_drvdata(dev); int ret, temp_mC; - ret = rv3029_read_temp(dev, &temp_mC); + ret = rv3029_read_temp(rv3029, &temp_mC); if (ret < 0) return ret; @@ -702,9 +612,10 @@ static ssize_t rv3029_hwmon_set_update_interval(struct device *dev, const char *buf, size_t count) { + struct rv3029_data *rv3029 = dev_get_drvdata(dev); + unsigned int th_set_bits = 0; unsigned long interval_ms; int ret; - u8 th_set_bits = 0; ret = kstrtoul(buf, 10, &interval_ms); if (ret < 0) @@ -715,7 +626,7 @@ static ssize_t rv3029_hwmon_set_update_interval(struct device *dev, if (interval_ms >= 16000) th_set_bits |= RV3029_EECTRL_THP; } - ret = rv3029_eeprom_update_bits(dev, RV3029_CONTROL_E2P_EECTRL, + ret = rv3029_eeprom_update_bits(rv3029, RV3029_CONTROL_E2P_EECTRL, RV3029_EECTRL_THE | RV3029_EECTRL_THP, th_set_bits); if (ret < 0) @@ -728,10 +639,11 @@ static ssize_t rv3029_hwmon_show_update_interval(struct device *dev, struct device_attribute *attr, char *buf) { + struct rv3029_data *rv3029 = dev_get_drvdata(dev); int ret, interval_ms; u8 eectrl; - ret = rv3029_eeprom_read(dev, RV3029_CONTROL_E2P_EECTRL, + ret = rv3029_eeprom_read(rv3029, RV3029_CONTROL_E2P_EECTRL, &eectrl, 1); if (ret < 0) return ret; @@ -785,14 +697,23 @@ static void rv3029_hwmon_register(struct device *dev, const char *name) static struct rtc_class_ops rv3029_rtc_ops = { .read_time = rv3029_read_time, .set_time = rv3029_set_time, + .ioctl = rv3029_ioctl, }; static int rv3029_probe(struct device *dev, struct regmap *regmap, int irq, const char *name) { struct rv3029_data *rv3029; + struct nvmem_config nvmem_cfg = { + .name = "rv3029_nvram", + .word_size = 1, + .stride = 1, + .size = RV3029_RAM_SECTION_LEN, + .type = NVMEM_TYPE_BATTERY_BACKED, + .reg_read = rv3029_nvram_read, + .reg_write = rv3029_nvram_write, + }; int rc = 0; - u8 buf[1]; rv3029 = devm_kzalloc(dev, sizeof(*rv3029), GFP_KERNEL); if (!rv3029) @@ -803,21 +724,12 @@ static int rv3029_probe(struct device *dev, struct regmap *regmap, int irq, rv3029->dev = dev; dev_set_drvdata(dev, rv3029); - rc = rv3029_get_sr(dev, buf); - if (rc < 0) { - dev_err(dev, "reading status failed\n"); - return rc; - } - rv3029_trickle_config(dev); rv3029_hwmon_register(dev, name); - rv3029->rtc = devm_rtc_device_register(dev, name, &rv3029_rtc_ops, - THIS_MODULE); - if (IS_ERR(rv3029->rtc)) { - dev_err(dev, "unable to register the class device\n"); + rv3029->rtc = devm_rtc_allocate_device(dev); + if (IS_ERR(rv3029->rtc)) return PTR_ERR(rv3029->rtc); - } if (rv3029->irq > 0) { rc = devm_request_threaded_irq(dev, rv3029->irq, @@ -834,20 +746,48 @@ static int rv3029_probe(struct device *dev, struct regmap *regmap, int irq, } } + rv3029->rtc->ops = &rv3029_rtc_ops; + rv3029->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; + rv3029->rtc->range_max = RTC_TIMESTAMP_END_2079; + + rc = rtc_register_device(rv3029->rtc); + if (rc) + return rc; + + nvmem_cfg.priv = rv3029->regmap; + rtc_nvmem_register(rv3029->rtc, &nvmem_cfg); + return 0; } +static const struct regmap_range rv3029_holes_range[] = { + regmap_reg_range(0x05, 0x07), + regmap_reg_range(0x0f, 0x0f), + regmap_reg_range(0x17, 0x17), + regmap_reg_range(0x1a, 0x1f), + regmap_reg_range(0x21, 0x27), + regmap_reg_range(0x34, 0x37), +}; + +static const struct regmap_access_table rv3029_regs = { + .no_ranges = rv3029_holes_range, + .n_no_ranges = ARRAY_SIZE(rv3029_holes_range), +}; + +static const struct regmap_config config = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = &rv3029_regs, + .wr_table = &rv3029_regs, + .max_register = 0x3f, +}; + #if IS_ENABLED(CONFIG_I2C) static int rv3029_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct regmap *regmap; - static const struct regmap_config config = { - .reg_bits = 8, - .val_bits = 8, - }; - if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_I2C_BLOCK | I2C_FUNC_SMBUS_BYTE)) { dev_err(&client->dev, "Adapter does not support SMBUS_I2C_BLOCK or SMBUS_I2C_BYTE\n"); @@ -855,11 +795,8 @@ static int rv3029_i2c_probe(struct i2c_client *client, } regmap = devm_regmap_init_i2c(client, &config); - if (IS_ERR(regmap)) { - dev_err(&client->dev, "%s: regmap allocation failed: %ld\n", - __func__, PTR_ERR(regmap)); + if (IS_ERR(regmap)) return PTR_ERR(regmap); - } return rv3029_probe(&client->dev, regmap, client->irq, client->name); } @@ -873,24 +810,20 @@ MODULE_DEVICE_TABLE(i2c, rv3029_id); static const struct of_device_id rv3029_of_match[] = { { .compatible = "microcrystal,rv3029" }, - /* Backward compatibility only, do not use compatibles below: */ - { .compatible = "rv3029" }, - { .compatible = "rv3029c2" }, - { .compatible = "mc,rv3029c2" }, { } }; MODULE_DEVICE_TABLE(of, rv3029_of_match); static struct i2c_driver rv3029_driver = { .driver = { - .name = "rtc-rv3029c2", + .name = "rv3029", .of_match_table = of_match_ptr(rv3029_of_match), }, .probe = rv3029_i2c_probe, .id_table = rv3029_id, }; -static int rv3029_register_driver(void) +static int __init rv3029_register_driver(void) { return i2c_add_driver(&rv3029_driver); } @@ -902,7 +835,7 @@ static void rv3029_unregister_driver(void) #else -static int rv3029_register_driver(void) +static int __init rv3029_register_driver(void) { return 0; } @@ -917,18 +850,11 @@ static void rv3029_unregister_driver(void) static int rv3049_probe(struct spi_device *spi) { - static const struct regmap_config config = { - .reg_bits = 8, - .val_bits = 8, - }; struct regmap *regmap; regmap = devm_regmap_init_spi(spi, &config); - if (IS_ERR(regmap)) { - dev_err(&spi->dev, "%s: regmap allocation failed: %ld\n", - __func__, PTR_ERR(regmap)); + if (IS_ERR(regmap)) return PTR_ERR(regmap); - } return rv3029_probe(&spi->dev, regmap, spi->irq, "rv3049"); } @@ -940,24 +866,24 @@ static struct spi_driver rv3049_driver = { .probe = rv3049_probe, }; -static int rv3049_register_driver(void) +static int __init rv3049_register_driver(void) { return spi_register_driver(&rv3049_driver); } -static void rv3049_unregister_driver(void) +static void __exit rv3049_unregister_driver(void) { spi_unregister_driver(&rv3049_driver); } #else -static int rv3049_register_driver(void) +static int __init rv3049_register_driver(void) { return 0; } -static void rv3049_unregister_driver(void) +static void __exit rv3049_unregister_driver(void) { } @@ -968,16 +894,12 @@ static int __init rv30x9_init(void) int ret; ret = rv3029_register_driver(); - if (ret) { - pr_err("Failed to register rv3029 driver: %d\n", ret); + if (ret) return ret; - } ret = rv3049_register_driver(); - if (ret) { - pr_err("Failed to register rv3049 driver: %d\n", ret); + if (ret) rv3029_unregister_driver(); - } return ret; } diff --git a/drivers/rtc/rtc-rv8803.c b/drivers/rtc/rtc-rv8803.c index 4960f0a2b249..93c3a6b627bd 100644 --- a/drivers/rtc/rtc-rv8803.c +++ b/drivers/rtc/rtc-rv8803.c @@ -411,6 +411,7 @@ static int rv8803_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) { struct i2c_client *client = to_i2c_client(dev); struct rv8803_data *rv8803 = dev_get_drvdata(dev); + unsigned int vl = 0; int flags, ret = 0; switch (cmd) { @@ -419,18 +420,15 @@ static int rv8803_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) if (flags < 0) return flags; - if (flags & RV8803_FLAG_V1F) + if (flags & RV8803_FLAG_V1F) { dev_warn(&client->dev, "Voltage low, temperature compensation stopped.\n"); + vl = RTC_VL_ACCURACY_LOW; + } if (flags & RV8803_FLAG_V2F) - dev_warn(&client->dev, "Voltage low, data loss detected.\n"); - - flags &= RV8803_FLAG_V1F | RV8803_FLAG_V2F; + vl |= RTC_VL_DATA_INVALID; - if (copy_to_user((void __user *)arg, &flags, sizeof(int))) - return -EFAULT; - - return 0; + return put_user(vl, (unsigned int __user *)arg); case RTC_VL_CLR: mutex_lock(&rv8803->flags_lock); @@ -440,7 +438,7 @@ static int rv8803_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) return flags; } - flags &= ~(RV8803_FLAG_V1F | RV8803_FLAG_V2F); + flags &= ~RV8803_FLAG_V1F; ret = rv8803_write_reg(client, RV8803_FLAG, flags); mutex_unlock(&rv8803->flags_lock); if (ret) diff --git a/drivers/rtc/rtc-rx8010.c b/drivers/rtc/rtc-rx8010.c index 8102469e27c0..fe010151ec8f 100644 --- a/drivers/rtc/rtc-rx8010.c +++ b/drivers/rtc/rtc-rx8010.c @@ -389,9 +389,8 @@ static int rx8010_alarm_irq_enable(struct device *dev, static int rx8010_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) { - struct i2c_client *client = to_i2c_client(dev); struct rx8010_data *rx8010 = dev_get_drvdata(dev); - int ret, tmp; + int tmp; int flagreg; switch (cmd) { @@ -400,24 +399,8 @@ static int rx8010_ioctl(struct device *dev, unsigned int cmd, unsigned long arg) if (flagreg < 0) return flagreg; - tmp = !!(flagreg & RX8010_FLAG_VLF); - if (copy_to_user((void __user *)arg, &tmp, sizeof(int))) - return -EFAULT; - - return 0; - - case RTC_VL_CLR: - flagreg = i2c_smbus_read_byte_data(rx8010->client, RX8010_FLAG); - if (flagreg < 0) { - return flagreg; - } - - flagreg &= ~RX8010_FLAG_VLF; - ret = i2c_smbus_write_byte_data(client, RX8010_FLAG, flagreg); - if (ret < 0) - return ret; - - return 0; + tmp = flagreg & RX8010_FLAG_VLF ? RTC_VL_DATA_INVALID : 0; + return put_user(tmp, (unsigned int __user *)arg); default: return -ENOIOCTLCMD; @@ -482,7 +465,7 @@ static int rx8010_probe(struct i2c_client *client, rx8010->rtc->max_user_freq = 1; - return err; + return 0; } static struct i2c_driver rx8010_driver = { diff --git a/drivers/rtc/rtc-rx8025.c b/drivers/rtc/rtc-rx8025.c index b9bda10589e0..a24f85893f90 100644 --- a/drivers/rtc/rtc-rx8025.c +++ b/drivers/rtc/rtc-rx8025.c @@ -67,7 +67,6 @@ static const struct i2c_device_id rx8025_id[] = { MODULE_DEVICE_TABLE(i2c, rx8025_id); struct rx8025_data { - struct i2c_client *client; struct rtc_device *rtc; u8 ctrl1; }; @@ -103,10 +102,10 @@ static s32 rx8025_write_regs(const struct i2c_client *client, static int rx8025_check_validity(struct device *dev) { - struct rx8025_data *rx8025 = dev_get_drvdata(dev); + struct i2c_client *client = to_i2c_client(dev); int ctrl2; - ctrl2 = rx8025_read_reg(rx8025->client, RX8025_REG_CTRL2); + ctrl2 = rx8025_read_reg(client, RX8025_REG_CTRL2); if (ctrl2 < 0) return ctrl2; @@ -178,6 +177,7 @@ out: static int rx8025_get_time(struct device *dev, struct rtc_time *dt) { + struct i2c_client *client = to_i2c_client(dev); struct rx8025_data *rx8025 = dev_get_drvdata(dev); u8 date[7]; int err; @@ -186,7 +186,7 @@ static int rx8025_get_time(struct device *dev, struct rtc_time *dt) if (err) return err; - err = rx8025_read_regs(rx8025->client, RX8025_REG_SEC, 7, date); + err = rx8025_read_regs(client, RX8025_REG_SEC, 7, date); if (err) return err; @@ -211,6 +211,7 @@ static int rx8025_get_time(struct device *dev, struct rtc_time *dt) static int rx8025_set_time(struct device *dev, struct rtc_time *dt) { + struct i2c_client *client = to_i2c_client(dev); struct rx8025_data *rx8025 = dev_get_drvdata(dev); u8 date[7]; int ret; @@ -237,11 +238,11 @@ static int rx8025_set_time(struct device *dev, struct rtc_time *dt) dev_dbg(dev, "%s: write %7ph\n", __func__, date); - ret = rx8025_write_regs(rx8025->client, RX8025_REG_SEC, 7, date); + ret = rx8025_write_regs(client, RX8025_REG_SEC, 7, date); if (ret < 0) return ret; - return rx8025_reset_validity(rx8025->client); + return rx8025_reset_validity(client); } static int rx8025_init_client(struct i2c_client *client) @@ -251,7 +252,7 @@ static int rx8025_init_client(struct i2c_client *client) int need_clear = 0; int err; - err = rx8025_read_regs(rx8025->client, RX8025_REG_CTRL1, 2, ctrl); + err = rx8025_read_regs(client, RX8025_REG_CTRL1, 2, ctrl); if (err) goto out; @@ -280,8 +281,8 @@ out: /* Alarm support */ static int rx8025_read_alarm(struct device *dev, struct rtc_wkalrm *t) { + struct i2c_client *client = to_i2c_client(dev); struct rx8025_data *rx8025 = dev_get_drvdata(dev); - struct i2c_client *client = rx8025->client; u8 ald[2]; int ctrl2, err; @@ -347,18 +348,18 @@ static int rx8025_set_alarm(struct device *dev, struct rtc_wkalrm *t) if (rx8025->ctrl1 & RX8025_BIT_CTRL1_DALE) { rx8025->ctrl1 &= ~RX8025_BIT_CTRL1_DALE; - err = rx8025_write_reg(rx8025->client, RX8025_REG_CTRL1, + err = rx8025_write_reg(client, RX8025_REG_CTRL1, rx8025->ctrl1); if (err) return err; } - err = rx8025_write_regs(rx8025->client, RX8025_REG_ALDMIN, 2, ald); + err = rx8025_write_regs(client, RX8025_REG_ALDMIN, 2, ald); if (err) return err; if (t->enabled) { rx8025->ctrl1 |= RX8025_BIT_CTRL1_DALE; - err = rx8025_write_reg(rx8025->client, RX8025_REG_CTRL1, + err = rx8025_write_reg(client, RX8025_REG_CTRL1, rx8025->ctrl1); if (err) return err; @@ -369,6 +370,7 @@ static int rx8025_set_alarm(struct device *dev, struct rtc_wkalrm *t) static int rx8025_alarm_irq_enable(struct device *dev, unsigned int enabled) { + struct i2c_client *client = to_i2c_client(dev); struct rx8025_data *rx8025 = dev_get_drvdata(dev); u8 ctrl1; int err; @@ -381,7 +383,7 @@ static int rx8025_alarm_irq_enable(struct device *dev, unsigned int enabled) if (ctrl1 != rx8025->ctrl1) { rx8025->ctrl1 = ctrl1; - err = rx8025_write_reg(rx8025->client, RX8025_REG_CTRL1, + err = rx8025_write_reg(client, RX8025_REG_CTRL1, rx8025->ctrl1); if (err) return err; @@ -516,7 +518,6 @@ static int rx8025_probe(struct i2c_client *client, if (!rx8025) return -ENOMEM; - rx8025->client = client; i2c_set_clientdata(client, rx8025); err = rx8025_init_client(client); diff --git a/drivers/rtc/rtc-stm32.c b/drivers/rtc/rtc-stm32.c index 781cabb2afca..d774aa18f57a 100644 --- a/drivers/rtc/rtc-stm32.c +++ b/drivers/rtc/rtc-stm32.c @@ -897,8 +897,11 @@ static int stm32_rtc_resume(struct device *dev) } ret = stm32_rtc_wait_sync(rtc); - if (ret < 0) + if (ret < 0) { + if (rtc->data->has_pclk) + clk_disable_unprepare(rtc->pclk); return ret; + } if (device_may_wakeup(dev)) return disable_irq_wake(rtc->irq_alarm); diff --git a/drivers/rtc/rtc-tps6586x.c b/drivers/rtc/rtc-tps6586x.c index 859d901fa6cb..e39af2d67051 100644 --- a/drivers/rtc/rtc-tps6586x.c +++ b/drivers/rtc/rtc-tps6586x.c @@ -23,6 +23,7 @@ #include <linux/device.h> #include <linux/err.h> #include <linux/init.h> +#include <linux/irq.h> #include <linux/kernel.h> #include <linux/mfd/tps6586x.h> #include <linux/module.h> @@ -267,6 +268,8 @@ static int tps6586x_rtc_probe(struct platform_device *pdev) rtc->rtc->start_secs = mktime64(2009, 1, 1, 0, 0, 0); rtc->rtc->set_start_time = true; + irq_set_status_flags(rtc->irq, IRQ_NOAUTOEN); + ret = devm_request_threaded_irq(&pdev->dev, rtc->irq, NULL, tps6586x_rtc_irq, IRQF_ONESHOT, @@ -276,7 +279,6 @@ static int tps6586x_rtc_probe(struct platform_device *pdev) rtc->irq, ret); goto fail_rtc_register; } - disable_irq(rtc->irq); ret = rtc_register_device(rtc->rtc); if (ret) diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c index 539690568298..5786866c09e9 100644 --- a/drivers/rtc/rtc-zynqmp.c +++ b/drivers/rtc/rtc-zynqmp.c @@ -94,7 +94,7 @@ static int xlnx_rtc_read_time(struct device *dev, struct rtc_time *tm) * RTC has updated the CURRENT_TIME with the time written into * SET_TIME_WRITE register. */ - rtc_time64_to_tm(readl(xrtcdev->reg_base + RTC_CUR_TM), tm); + read_time = readl(xrtcdev->reg_base + RTC_CUR_TM); } else { /* * Time written in SET_TIME_WRITE has not yet updated into @@ -104,8 +104,8 @@ static int xlnx_rtc_read_time(struct device *dev, struct rtc_time *tm) * reading. */ read_time = readl(xrtcdev->reg_base + RTC_SET_TM_RD) - 1; - rtc_time64_to_tm(read_time, tm); } + rtc_time64_to_tm(read_time, tm); return 0; } diff --git a/drivers/s390/block/dasd_proc.c b/drivers/s390/block/dasd_proc.c index 8d4d69ea5baf..62a859ea67f8 100644 --- a/drivers/s390/block/dasd_proc.c +++ b/drivers/s390/block/dasd_proc.c @@ -320,13 +320,12 @@ out_error: #endif /* CONFIG_DASD_PROFILE */ } -static const struct file_operations dasd_stats_proc_fops = { - .owner = THIS_MODULE, - .open = dasd_stats_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = dasd_stats_proc_write, +static const struct proc_ops dasd_stats_proc_ops = { + .proc_open = dasd_stats_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = dasd_stats_proc_write, }; /* @@ -347,7 +346,7 @@ dasd_proc_init(void) dasd_statistics_entry = proc_create("statistics", S_IFREG | S_IRUGO | S_IWUSR, dasd_proc_root_entry, - &dasd_stats_proc_fops); + &dasd_stats_proc_ops); if (!dasd_statistics_entry) goto out_nostatistics; return 0; diff --git a/drivers/s390/cio/blacklist.c b/drivers/s390/cio/blacklist.c index 2a3f874a21d5..da642e811f7f 100644 --- a/drivers/s390/cio/blacklist.c +++ b/drivers/s390/cio/blacklist.c @@ -398,12 +398,12 @@ cio_ignore_proc_open(struct inode *inode, struct file *file) sizeof(struct ccwdev_iter)); } -static const struct file_operations cio_ignore_proc_fops = { - .open = cio_ignore_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release_private, - .write = cio_ignore_write, +static const struct proc_ops cio_ignore_proc_ops = { + .proc_open = cio_ignore_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = seq_release_private, + .proc_write = cio_ignore_write, }; static int @@ -412,7 +412,7 @@ cio_ignore_proc_init (void) struct proc_dir_entry *entry; entry = proc_create("cio_ignore", S_IFREG | S_IRUGO | S_IWUSR, NULL, - &cio_ignore_proc_fops); + &cio_ignore_proc_ops); if (!entry) return -ENOENT; return 0; diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 831850435c23..94edbb33d0d1 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -1372,18 +1372,17 @@ static ssize_t cio_settle_write(struct file *file, const char __user *buf, return ret ? ret : count; } -static const struct file_operations cio_settle_proc_fops = { - .open = nonseekable_open, - .write = cio_settle_write, - .llseek = no_llseek, +static const struct proc_ops cio_settle_proc_ops = { + .proc_open = nonseekable_open, + .proc_write = cio_settle_write, + .proc_lseek = no_llseek, }; static int __init cio_settle_init(void) { struct proc_dir_entry *entry; - entry = proc_create("cio_settle", S_IWUSR, NULL, - &cio_settle_proc_fops); + entry = proc_create("cio_settle", S_IWUSR, NULL, &cio_settle_proc_ops); if (!entry) return -ENOMEM; return 0; diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile index 52aa95c8af4b..22d2db690cd3 100644 --- a/drivers/s390/crypto/Makefile +++ b/drivers/s390/crypto/Makefile @@ -7,7 +7,8 @@ ap-objs := ap_bus.o ap_card.o ap_queue.o obj-$(subst m,y,$(CONFIG_ZCRYPT)) += ap.o # zcrypt_api.o and zcrypt_msgtype*.o depend on ap.o zcrypt-objs := zcrypt_api.o zcrypt_card.o zcrypt_queue.o -zcrypt-objs += zcrypt_msgtype6.o zcrypt_msgtype50.o zcrypt_ccamisc.o +zcrypt-objs += zcrypt_msgtype6.o zcrypt_msgtype50.o +zcrypt-objs += zcrypt_ccamisc.o zcrypt_ep11misc.o obj-$(CONFIG_ZCRYPT) += zcrypt.o # adapter drivers depend on ap.o and zcrypt.o obj-$(CONFIG_ZCRYPT) += zcrypt_cex2c.o zcrypt_cex2a.o zcrypt_cex4.o diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index d78d77686d7b..71dae64ba994 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -25,6 +25,7 @@ #include "zcrypt_api.h" #include "zcrypt_ccamisc.h" +#include "zcrypt_ep11misc.h" MODULE_LICENSE("GPL"); MODULE_AUTHOR("IBM Corporation"); @@ -71,6 +72,17 @@ struct protaeskeytoken { u8 protkey[MAXPROTKEYSIZE]; /* the protected key blob */ } __packed; +/* inside view of a clear key token (type 0x00 version 0x02) */ +struct clearaeskeytoken { + u8 type; /* 0x00 for PAES specific key tokens */ + u8 res0[3]; + u8 version; /* 0x02 for clear AES key token */ + u8 res1[3]; + u32 keytype; /* key type, one of the PKEY_KEYTYPE values */ + u32 len; /* bytes actually stored in clearkey[] */ + u8 clearkey[0]; /* clear key value */ +} __packed; + /* * Create a protected key from a clear key value. */ @@ -173,6 +185,72 @@ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) } /* + * Construct EP11 key with given clear key value. + */ +static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, + u8 *keybuf, size_t *keybuflen) +{ + int i, rc; + u16 card, dom; + u32 nr_apqns, *apqns = NULL; + + /* build a list of apqns suitable for ep11 keys with cpacf support */ + rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, EP11_API_V, NULL); + if (rc) + goto out; + + /* go through the list of apqns and try to bild an ep11 key */ + for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { + card = apqns[i] >> 16; + dom = apqns[i] & 0xFFFF; + rc = ep11_clr2keyblob(card, dom, clrkeylen * 8, + 0, clrkey, keybuf, keybuflen); + if (rc == 0) + break; + } + +out: + kfree(apqns); + if (rc) + DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + return rc; +} + +/* + * Find card and transform EP11 secure key into protected key. + */ +static int pkey_ep11key2pkey(const u8 *key, struct pkey_protkey *pkey) +{ + int i, rc; + u16 card, dom; + u32 nr_apqns, *apqns = NULL; + struct ep11keyblob *kb = (struct ep11keyblob *) key; + + /* build a list of apqns suitable for this key */ + rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, EP11_API_V, kb->wkvp); + if (rc) + goto out; + + /* go through the list of apqns and try to derive an pkey */ + for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { + card = apqns[i] >> 16; + dom = apqns[i] & 0xFFFF; + rc = ep11_key2protkey(card, dom, key, kb->head.len, + pkey->protkey, &pkey->len, &pkey->type); + if (rc == 0) + break; + } + +out: + kfree(apqns); + if (rc) + DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + return rc; +} + +/* * Verify key and give back some info about the key. */ static int pkey_verifykey(const struct pkey_seckey *seckey, @@ -305,26 +383,90 @@ static int pkey_verifyprotkey(const struct pkey_protkey *protkey) static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, struct pkey_protkey *protkey) { + int rc = -EINVAL; + u8 *tmpbuf = NULL; struct keytoken_header *hdr = (struct keytoken_header *)key; - struct protaeskeytoken *t; switch (hdr->version) { - case TOKVER_PROTECTED_KEY: - if (keylen != sizeof(struct protaeskeytoken)) - return -EINVAL; + case TOKVER_PROTECTED_KEY: { + struct protaeskeytoken *t; + if (keylen != sizeof(struct protaeskeytoken)) + goto out; t = (struct protaeskeytoken *)key; protkey->len = t->len; protkey->type = t->keytype; memcpy(protkey->protkey, t->protkey, sizeof(protkey->protkey)); - - return pkey_verifyprotkey(protkey); + rc = pkey_verifyprotkey(protkey); + break; + } + case TOKVER_CLEAR_KEY: { + struct clearaeskeytoken *t; + struct pkey_clrkey ckey; + union u_tmpbuf { + u8 skey[SECKEYBLOBSIZE]; + u8 ep11key[MAXEP11AESKEYBLOBSIZE]; + }; + size_t tmpbuflen = sizeof(union u_tmpbuf); + + if (keylen < sizeof(struct clearaeskeytoken)) + goto out; + t = (struct clearaeskeytoken *)key; + if (keylen != sizeof(*t) + t->len) + goto out; + if ((t->keytype == PKEY_KEYTYPE_AES_128 && t->len == 16) + || (t->keytype == PKEY_KEYTYPE_AES_192 && t->len == 24) + || (t->keytype == PKEY_KEYTYPE_AES_256 && t->len == 32)) + memcpy(ckey.clrkey, t->clearkey, t->len); + else + goto out; + /* alloc temp key buffer space */ + tmpbuf = kmalloc(tmpbuflen, GFP_ATOMIC); + if (!tmpbuf) { + rc = -ENOMEM; + goto out; + } + /* try direct way with the PCKMO instruction */ + rc = pkey_clr2protkey(t->keytype, &ckey, protkey); + if (rc == 0) + break; + /* PCKMO failed, so try the CCA secure key way */ + rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, + ckey.clrkey, tmpbuf); + if (rc == 0) + rc = pkey_skey2pkey(tmpbuf, protkey); + if (rc == 0) + break; + /* if the CCA way also failed, let's try via EP11 */ + rc = pkey_clr2ep11key(ckey.clrkey, t->len, + tmpbuf, &tmpbuflen); + if (rc == 0) + rc = pkey_ep11key2pkey(tmpbuf, protkey); + /* now we should really have an protected key */ + DEBUG_ERR("%s unable to build protected key from clear", + __func__); + break; + } + case TOKVER_EP11_AES: { + if (keylen < MINEP11AESKEYBLOBSIZE) + goto out; + /* check ep11 key for exportable as protected key */ + rc = ep11_check_aeskeyblob(debug_info, 3, key, 0, 1); + if (rc) + goto out; + rc = pkey_ep11key2pkey(key, protkey); + break; + } default: DEBUG_ERR("%s unknown/unsupported non-CCA token version %d\n", __func__, hdr->version); - return -EINVAL; + rc = -EINVAL; } + +out: + kfree(tmpbuf); + return rc; } /* @@ -403,6 +545,10 @@ static int pkey_genseckey2(const struct pkey_apqn *apqns, size_t nr_apqns, if (*keybufsize < SECKEYBLOBSIZE) return -EINVAL; break; + case PKEY_TYPE_EP11: + if (*keybufsize < MINEP11AESKEYBLOBSIZE) + return -EINVAL; + break; default: return -EINVAL; } @@ -419,7 +565,10 @@ static int pkey_genseckey2(const struct pkey_apqn *apqns, size_t nr_apqns, for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { card = apqns[i].card; dom = apqns[i].domain; - if (ktype == PKEY_TYPE_CCA_DATA) { + if (ktype == PKEY_TYPE_EP11) { + rc = ep11_genaeskey(card, dom, ksize, kflags, + keybuf, keybufsize); + } else if (ktype == PKEY_TYPE_CCA_DATA) { rc = cca_genseckey(card, dom, ksize, keybuf); *keybufsize = (rc ? 0 : SECKEYBLOBSIZE); } else /* TOKVER_CCA_VLSC */ @@ -450,6 +599,10 @@ static int pkey_clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, if (*keybufsize < SECKEYBLOBSIZE) return -EINVAL; break; + case PKEY_TYPE_EP11: + if (*keybufsize < MINEP11AESKEYBLOBSIZE) + return -EINVAL; + break; default: return -EINVAL; } @@ -466,7 +619,10 @@ static int pkey_clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { card = apqns[i].card; dom = apqns[i].domain; - if (ktype == PKEY_TYPE_CCA_DATA) { + if (ktype == PKEY_TYPE_EP11) { + rc = ep11_clr2keyblob(card, dom, ksize, kflags, + clrkey, keybuf, keybufsize); + } else if (ktype == PKEY_TYPE_CCA_DATA) { rc = cca_clr2seckey(card, dom, ksize, clrkey, keybuf); *keybufsize = (rc ? 0 : SECKEYBLOBSIZE); @@ -489,11 +645,11 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, u32 _nr_apqns, *_apqns = NULL; struct keytoken_header *hdr = (struct keytoken_header *)key; - if (keylen < sizeof(struct keytoken_header) || - hdr->type != TOKTYPE_CCA_INTERNAL) + if (keylen < sizeof(struct keytoken_header)) return -EINVAL; - if (hdr->version == TOKVER_CCA_AES) { + if (hdr->type == TOKTYPE_CCA_INTERNAL + && hdr->version == TOKVER_CCA_AES) { struct secaeskeytoken *t = (struct secaeskeytoken *)key; rc = cca_check_secaeskeytoken(debug_info, 3, key, 0); @@ -521,7 +677,8 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, *cardnr = ((struct pkey_apqn *)_apqns)->card; *domain = ((struct pkey_apqn *)_apqns)->domain; - } else if (hdr->version == TOKVER_CCA_VLSC) { + } else if (hdr->type == TOKTYPE_CCA_INTERNAL + && hdr->version == TOKVER_CCA_VLSC) { struct cipherkeytoken *t = (struct cipherkeytoken *)key; rc = cca_check_secaescipherkey(debug_info, 3, key, 0, 1); @@ -556,6 +713,29 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, *cardnr = ((struct pkey_apqn *)_apqns)->card; *domain = ((struct pkey_apqn *)_apqns)->domain; + } else if (hdr->type == TOKTYPE_NON_CCA + && hdr->version == TOKVER_EP11_AES) { + struct ep11keyblob *kb = (struct ep11keyblob *)key; + + rc = ep11_check_aeskeyblob(debug_info, 3, key, 0, 1); + if (rc) + goto out; + if (ktype) + *ktype = PKEY_TYPE_EP11; + if (ksize) + *ksize = kb->head.keybitlen; + + rc = ep11_findcard2(&_apqns, &_nr_apqns, *cardnr, *domain, + ZCRYPT_CEX7, EP11_API_V, kb->wkvp); + if (rc) + goto out; + + if (flags) + *flags = PKEY_FLAGS_MATCH_CUR_MKVP; + + *cardnr = ((struct pkey_apqn *)_apqns)->card; + *domain = ((struct pkey_apqn *)_apqns)->domain; + } else rc = -EINVAL; @@ -578,30 +758,32 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, if (keylen < sizeof(struct keytoken_header)) return -EINVAL; - switch (hdr->type) { - case TOKTYPE_NON_CCA: - return pkey_nonccatok2pkey(key, keylen, pkey); - case TOKTYPE_CCA_INTERNAL: - switch (hdr->version) { - case TOKVER_CCA_AES: + if (hdr->type == TOKTYPE_CCA_INTERNAL) { + if (hdr->version == TOKVER_CCA_AES) { if (keylen != sizeof(struct secaeskeytoken)) return -EINVAL; if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) return -EINVAL; - break; - case TOKVER_CCA_VLSC: + } else if (hdr->version == TOKVER_CCA_VLSC) { if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) return -EINVAL; if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) return -EINVAL; - break; - default: + } else { DEBUG_ERR("%s unknown CCA internal token version %d\n", __func__, hdr->version); return -EINVAL; } - break; - default: + } else if (hdr->type == TOKTYPE_NON_CCA) { + if (hdr->version == TOKVER_EP11_AES) { + if (keylen < sizeof(struct ep11keyblob)) + return -EINVAL; + if (ep11_check_aeskeyblob(debug_info, 3, key, 0, 1)) + return -EINVAL; + } else { + return pkey_nonccatok2pkey(key, keylen, pkey); + } + } else { DEBUG_ERR("%s unknown/unsupported blob type %d\n", __func__, hdr->type); return -EINVAL; @@ -611,12 +793,21 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { card = apqns[i].card; dom = apqns[i].domain; - if (hdr->version == TOKVER_CCA_AES) + if (hdr->type == TOKTYPE_CCA_INTERNAL + && hdr->version == TOKVER_CCA_AES) rc = cca_sec2protkey(card, dom, key, pkey->protkey, &pkey->len, &pkey->type); - else /* TOKVER_CCA_VLSC */ + else if (hdr->type == TOKTYPE_CCA_INTERNAL + && hdr->version == TOKVER_CCA_VLSC) rc = cca_cipher2protkey(card, dom, key, pkey->protkey, &pkey->len, &pkey->type); + else { /* EP11 AES secure key blob */ + struct ep11keyblob *kb = (struct ep11keyblob *) key; + + rc = ep11_key2protkey(card, dom, key, kb->head.len, + pkey->protkey, &pkey->len, + &pkey->type); + } if (rc == 0) break; } @@ -631,12 +822,24 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, u32 _nr_apqns, *_apqns = NULL; struct keytoken_header *hdr = (struct keytoken_header *)key; - if (keylen < sizeof(struct keytoken_header) || - hdr->type != TOKTYPE_CCA_INTERNAL || - flags == 0) + if (keylen < sizeof(struct keytoken_header) || flags == 0) return -EINVAL; - if (hdr->version == TOKVER_CCA_AES || hdr->version == TOKVER_CCA_VLSC) { + if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_AES) { + int minhwtype = 0, api = 0; + struct ep11keyblob *kb = (struct ep11keyblob *) key; + + if (flags != PKEY_FLAGS_MATCH_CUR_MKVP) + return -EINVAL; + if (kb->attr & EP11_BLOB_PKEY_EXTRACTABLE) { + minhwtype = ZCRYPT_CEX7; + api = EP11_API_V; + } + rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + minhwtype, api, kb->wkvp); + if (rc) + goto out; + } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { int minhwtype = ZCRYPT_CEX3C; u64 cur_mkvp = 0, old_mkvp = 0; @@ -647,7 +850,7 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, cur_mkvp = t->mkvp; if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) old_mkvp = t->mkvp; - } else { + } else if (hdr->version == TOKVER_CCA_VLSC) { struct cipherkeytoken *t = (struct cipherkeytoken *)key; minhwtype = ZCRYPT_CEX6; @@ -655,19 +858,24 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, cur_mkvp = t->mkvp0; if (flags & PKEY_FLAGS_MATCH_ALT_MKVP) old_mkvp = t->mkvp0; + } else { + /* unknown cca internal token type */ + return -EINVAL; } rc = cca_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, minhwtype, cur_mkvp, old_mkvp, 1); if (rc) goto out; - if (apqns) { - if (*nr_apqns < _nr_apqns) - rc = -ENOSPC; - else - memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); - } - *nr_apqns = _nr_apqns; + } else + return -EINVAL; + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); } + *nr_apqns = _nr_apqns; out: kfree(_apqns); @@ -695,14 +903,26 @@ static int pkey_apqns4keytype(enum pkey_key_type ktype, minhwtype, cur_mkvp, old_mkvp, 1); if (rc) goto out; - if (apqns) { - if (*nr_apqns < _nr_apqns) - rc = -ENOSPC; - else - memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); - } - *nr_apqns = _nr_apqns; + } else if (ktype == PKEY_TYPE_EP11) { + u8 *wkvp = NULL; + + if (flags & PKEY_FLAGS_MATCH_CUR_MKVP) + wkvp = cur_mkvp; + rc = ep11_findcard2(&_apqns, &_nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, EP11_API_V, wkvp); + if (rc) + goto out; + + } else + return -EINVAL; + + if (apqns) { + if (*nr_apqns < _nr_apqns) + rc = -ENOSPC; + else + memcpy(apqns, _apqns, _nr_apqns * sizeof(u32)); } + *nr_apqns = _nr_apqns; out: kfree(_apqns); @@ -1357,8 +1577,9 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, bool is_xts, char *buf, loff_t off, size_t count) { - size_t keysize; - int rc; + int i, rc, card, dom; + u32 nr_apqns, *apqns = NULL; + size_t keysize = CCACIPHERTOKENSIZE; if (off != 0 || count < CCACIPHERTOKENSIZE) return -EINVAL; @@ -1366,22 +1587,31 @@ static ssize_t pkey_ccacipher_aes_attr_read(enum pkey_key_size keybits, if (count < 2 * CCACIPHERTOKENSIZE) return -EINVAL; - keysize = CCACIPHERTOKENSIZE; - rc = cca_gencipherkey(-1, -1, keybits, 0, buf, &keysize); + /* build a list of apqns able to generate an cipher key */ + rc = cca_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX6, 0, 0, 0); if (rc) return rc; - memset(buf + keysize, 0, CCACIPHERTOKENSIZE - keysize); - if (is_xts) { - keysize = CCACIPHERTOKENSIZE; - rc = cca_gencipherkey(-1, -1, keybits, 0, - buf + CCACIPHERTOKENSIZE, &keysize); + memset(buf, 0, is_xts ? 2 * keysize : keysize); + + /* simple try all apqns from the list */ + for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { + card = apqns[i] >> 16; + dom = apqns[i] & 0xFFFF; + rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); + if (rc == 0) + break; + } if (rc) return rc; - memset(buf + CCACIPHERTOKENSIZE + keysize, 0, - CCACIPHERTOKENSIZE - keysize); - return 2 * CCACIPHERTOKENSIZE; + if (is_xts) { + keysize = CCACIPHERTOKENSIZE; + buf += CCACIPHERTOKENSIZE; + rc = cca_gencipherkey(card, dom, keybits, 0, buf, &keysize); + if (rc == 0) + return 2 * CCACIPHERTOKENSIZE; } return CCACIPHERTOKENSIZE; @@ -1457,10 +1687,134 @@ static struct attribute_group ccacipher_attr_group = { .bin_attrs = ccacipher_attrs, }; +/* + * Sysfs attribute read function for all ep11 aes key binary attributes. + * The implementation can not deal with partial reads, because a new random + * secure key blob is generated with each read. In case of partial reads + * (i.e. off != 0 or count < key blob size) -EINVAL is returned. + * This function and the sysfs attributes using it provide EP11 key blobs + * padded to the upper limit of MAXEP11AESKEYBLOBSIZE which is currently + * 320 bytes. + */ +static ssize_t pkey_ep11_aes_attr_read(enum pkey_key_size keybits, + bool is_xts, char *buf, loff_t off, + size_t count) +{ + int i, rc, card, dom; + u32 nr_apqns, *apqns = NULL; + size_t keysize = MAXEP11AESKEYBLOBSIZE; + + if (off != 0 || count < MAXEP11AESKEYBLOBSIZE) + return -EINVAL; + if (is_xts) + if (count < 2 * MAXEP11AESKEYBLOBSIZE) + return -EINVAL; + + /* build a list of apqns able to generate an cipher key */ + rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, EP11_API_V, NULL); + if (rc) + return rc; + + memset(buf, 0, is_xts ? 2 * keysize : keysize); + + /* simple try all apqns from the list */ + for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { + card = apqns[i] >> 16; + dom = apqns[i] & 0xFFFF; + rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize); + if (rc == 0) + break; + } + if (rc) + return rc; + + if (is_xts) { + keysize = MAXEP11AESKEYBLOBSIZE; + buf += MAXEP11AESKEYBLOBSIZE; + rc = ep11_genaeskey(card, dom, keybits, 0, buf, &keysize); + if (rc == 0) + return 2 * MAXEP11AESKEYBLOBSIZE; + } + + return MAXEP11AESKEYBLOBSIZE; +} + +static ssize_t ep11_aes_128_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, false, buf, + off, count); +} + +static ssize_t ep11_aes_192_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_192, false, buf, + off, count); +} + +static ssize_t ep11_aes_256_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, false, buf, + off, count); +} + +static ssize_t ep11_aes_128_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_128, true, buf, + off, count); +} + +static ssize_t ep11_aes_256_xts_read(struct file *filp, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t off, + size_t count) +{ + return pkey_ep11_aes_attr_read(PKEY_SIZE_AES_256, true, buf, + off, count); +} + +static BIN_ATTR_RO(ep11_aes_128, MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_192, MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_256, MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_128_xts, 2 * MAXEP11AESKEYBLOBSIZE); +static BIN_ATTR_RO(ep11_aes_256_xts, 2 * MAXEP11AESKEYBLOBSIZE); + +static struct bin_attribute *ep11_attrs[] = { + &bin_attr_ep11_aes_128, + &bin_attr_ep11_aes_192, + &bin_attr_ep11_aes_256, + &bin_attr_ep11_aes_128_xts, + &bin_attr_ep11_aes_256_xts, + NULL +}; + +static struct attribute_group ep11_attr_group = { + .name = "ep11", + .bin_attrs = ep11_attrs, +}; + static const struct attribute_group *pkey_attr_groups[] = { &protkey_attr_group, &ccadata_attr_group, &ccacipher_attr_group, + &ep11_attr_group, NULL, }; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 9157e728a362..a42257d6c79e 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -36,6 +36,7 @@ #include "zcrypt_msgtype6.h" #include "zcrypt_msgtype50.h" #include "zcrypt_ccamisc.h" +#include "zcrypt_ep11misc.h" /* * Module description. @@ -849,7 +850,7 @@ static long _zcrypt_send_cprb(struct ap_perms *perms, /* check if device is online and eligible */ if (!zq->online || !zq->ops->send_cprb || - (tdom != (unsigned short) AUTOSELECT && + (tdom != AUTOSEL_DOM && tdom != AP_QID_QUEUE(zq->queue->qid))) continue; /* check if device node has admission for this queue */ @@ -874,7 +875,7 @@ static long _zcrypt_send_cprb(struct ap_perms *perms, /* in case of auto select, provide the correct domain */ qid = pref_zq->queue->qid; - if (*domain == (unsigned short) AUTOSELECT) + if (*domain == AUTOSEL_DOM) *domain = AP_QID_QUEUE(qid); rc = pref_zq->ops->send_cprb(pref_zq, xcRB, &ap_msg); @@ -901,7 +902,7 @@ static bool is_desired_ep11_card(unsigned int dev_id, struct ep11_target_dev *targets) { while (target_num-- > 0) { - if (dev_id == targets->ap_id) + if (targets->ap_id == dev_id || targets->ap_id == AUTOSEL_AP) return true; targets++; } @@ -912,16 +913,19 @@ static bool is_desired_ep11_queue(unsigned int dev_qid, unsigned short target_num, struct ep11_target_dev *targets) { + int card = AP_QID_CARD(dev_qid), dom = AP_QID_QUEUE(dev_qid); + while (target_num-- > 0) { - if (AP_MKQID(targets->ap_id, targets->dom_id) == dev_qid) + if ((targets->ap_id == card || targets->ap_id == AUTOSEL_AP) && + (targets->dom_id == dom || targets->dom_id == AUTOSEL_DOM)) return true; targets++; } return false; } -static long zcrypt_send_ep11_cprb(struct ap_perms *perms, - struct ep11_urb *xcrb) +static long _zcrypt_send_ep11_cprb(struct ap_perms *perms, + struct ep11_urb *xcrb) { struct zcrypt_card *zc, *pref_zc; struct zcrypt_queue *zq, *pref_zq; @@ -1026,6 +1030,12 @@ out: return rc; } +long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb) +{ + return _zcrypt_send_ep11_cprb(&ap_perms, xcrb); +} +EXPORT_SYMBOL(zcrypt_send_ep11_cprb); + static long zcrypt_rng(char *buffer) { struct zcrypt_card *zc, *pref_zc; @@ -1366,12 +1376,12 @@ static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd, if (copy_from_user(&xcrb, uxcrb, sizeof(xcrb))) return -EFAULT; do { - rc = zcrypt_send_ep11_cprb(perms, &xcrb); + rc = _zcrypt_send_ep11_cprb(perms, &xcrb); } while (rc == -EAGAIN); /* on failure: retry once again after a requested rescan */ if ((rc == -ENODEV) && (zcrypt_process_rescan())) do { - rc = zcrypt_send_ep11_cprb(perms, &xcrb); + rc = _zcrypt_send_ep11_cprb(perms, &xcrb); } while (rc == -EAGAIN); if (rc) ZCRYPT_DBF(DBF_DEBUG, "ioctl ZSENDEP11CPRB rc=%d\n", rc); @@ -1885,6 +1895,7 @@ void __exit zcrypt_api_exit(void) zcrypt_msgtype6_exit(); zcrypt_msgtype50_exit(); zcrypt_ccamisc_exit(); + zcrypt_ep11misc_exit(); zcrypt_debug_exit(); } diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index d464618cd84f..599e68bf53f7 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -140,6 +140,7 @@ struct zcrypt_ops *zcrypt_msgtype(unsigned char *, int); int zcrypt_api_init(void); void zcrypt_api_exit(void); long zcrypt_send_cprb(struct ica_xcRB *xcRB); +long zcrypt_send_ep11_cprb(struct ep11_urb *urb); void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus); int zcrypt_device_status_ext(int card, int queue, struct zcrypt_device_status_ext *devstatus); diff --git a/drivers/s390/crypto/zcrypt_ccamisc.h b/drivers/s390/crypto/zcrypt_ccamisc.h index 77b6cc7b8f82..3a9876d5ab0e 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.h +++ b/drivers/s390/crypto/zcrypt_ccamisc.h @@ -19,6 +19,7 @@ /* For TOKTYPE_NON_CCA: */ #define TOKVER_PROTECTED_KEY 0x01 /* Protected key token */ +#define TOKVER_CLEAR_KEY 0x02 /* Clear key token */ /* For TOKTYPE_CCA_INTERNAL: */ #define TOKVER_CCA_AES 0x04 /* CCA AES key token */ diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c index 6fabc906114c..9a9d02e19774 100644 --- a/drivers/s390/crypto/zcrypt_cex4.c +++ b/drivers/s390/crypto/zcrypt_cex4.c @@ -19,6 +19,7 @@ #include "zcrypt_error.h" #include "zcrypt_cex4.h" #include "zcrypt_ccamisc.h" +#include "zcrypt_ep11misc.h" #define CEX4A_MIN_MOD_SIZE 1 /* 8 bits */ #define CEX4A_MAX_MOD_SIZE_2K 256 /* 2048 bits */ @@ -71,11 +72,11 @@ static struct ap_device_id zcrypt_cex4_queue_ids[] = { MODULE_DEVICE_TABLE(ap, zcrypt_cex4_queue_ids); /* - * CCA card addditional device attributes + * CCA card additional device attributes */ -static ssize_t serialnr_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t cca_serialnr_show(struct device *dev, + struct device_attribute *attr, + char *buf) { struct cca_info ci; struct ap_card *ac = to_ap_card(dev); @@ -88,23 +89,25 @@ static ssize_t serialnr_show(struct device *dev, return snprintf(buf, PAGE_SIZE, "%s\n", ci.serial); } -static DEVICE_ATTR_RO(serialnr); + +static struct device_attribute dev_attr_cca_serialnr = + __ATTR(serialnr, 0444, cca_serialnr_show, NULL); static struct attribute *cca_card_attrs[] = { - &dev_attr_serialnr.attr, + &dev_attr_cca_serialnr.attr, NULL, }; -static const struct attribute_group cca_card_attr_group = { +static const struct attribute_group cca_card_attr_grp = { .attrs = cca_card_attrs, }; -/* - * CCA queue addditional device attributes - */ -static ssize_t mkvps_show(struct device *dev, - struct device_attribute *attr, - char *buf) + /* + * CCA queue additional device attributes + */ +static ssize_t cca_mkvps_show(struct device *dev, + struct device_attribute *attr, + char *buf) { int n = 0; struct cca_info ci; @@ -138,17 +141,233 @@ static ssize_t mkvps_show(struct device *dev, return n; } -static DEVICE_ATTR_RO(mkvps); + +static struct device_attribute dev_attr_cca_mkvps = + __ATTR(mkvps, 0444, cca_mkvps_show, NULL); static struct attribute *cca_queue_attrs[] = { - &dev_attr_mkvps.attr, + &dev_attr_cca_mkvps.attr, NULL, }; -static const struct attribute_group cca_queue_attr_group = { +static const struct attribute_group cca_queue_attr_grp = { .attrs = cca_queue_attrs, }; +/* + * EP11 card additional device attributes + */ +static ssize_t ep11_api_ordinalnr_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ep11_card_info ci; + struct ap_card *ac = to_ap_card(dev); + struct zcrypt_card *zc = ac->private; + + memset(&ci, 0, sizeof(ci)); + + ep11_get_card_info(ac->id, &ci, zc->online); + + if (ci.API_ord_nr > 0) + return snprintf(buf, PAGE_SIZE, "%u\n", ci.API_ord_nr); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static struct device_attribute dev_attr_ep11_api_ordinalnr = + __ATTR(API_ordinalnr, 0444, ep11_api_ordinalnr_show, NULL); + +static ssize_t ep11_fw_version_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ep11_card_info ci; + struct ap_card *ac = to_ap_card(dev); + struct zcrypt_card *zc = ac->private; + + memset(&ci, 0, sizeof(ci)); + + ep11_get_card_info(ac->id, &ci, zc->online); + + if (ci.FW_version > 0) + return snprintf(buf, PAGE_SIZE, "%d.%d\n", + (int)(ci.FW_version >> 8), + (int)(ci.FW_version & 0xFF)); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static struct device_attribute dev_attr_ep11_fw_version = + __ATTR(FW_version, 0444, ep11_fw_version_show, NULL); + +static ssize_t ep11_serialnr_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ep11_card_info ci; + struct ap_card *ac = to_ap_card(dev); + struct zcrypt_card *zc = ac->private; + + memset(&ci, 0, sizeof(ci)); + + ep11_get_card_info(ac->id, &ci, zc->online); + + if (ci.serial[0]) + return snprintf(buf, PAGE_SIZE, "%16.16s\n", ci.serial); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static struct device_attribute dev_attr_ep11_serialnr = + __ATTR(serialnr, 0444, ep11_serialnr_show, NULL); + +static const struct { + int mode_bit; + const char *mode_txt; +} ep11_op_modes[] = { + { 0, "FIPS2009" }, + { 1, "BSI2009" }, + { 2, "FIPS2011" }, + { 3, "BSI2011" }, + { 6, "BSICC2017" }, + { 0, NULL } +}; + +static ssize_t ep11_card_op_modes_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int i, n = 0; + struct ep11_card_info ci; + struct ap_card *ac = to_ap_card(dev); + struct zcrypt_card *zc = ac->private; + + memset(&ci, 0, sizeof(ci)); + + ep11_get_card_info(ac->id, &ci, zc->online); + + for (i = 0; ep11_op_modes[i].mode_txt; i++) { + if (ci.op_mode & (1 << ep11_op_modes[i].mode_bit)) { + if (n > 0) + buf[n++] = ' '; + n += snprintf(buf + n, PAGE_SIZE - n, + "%s", ep11_op_modes[i].mode_txt); + } + } + n += snprintf(buf + n, PAGE_SIZE - n, "\n"); + + return n; +} + +static struct device_attribute dev_attr_ep11_card_op_modes = + __ATTR(op_modes, 0444, ep11_card_op_modes_show, NULL); + +static struct attribute *ep11_card_attrs[] = { + &dev_attr_ep11_api_ordinalnr.attr, + &dev_attr_ep11_fw_version.attr, + &dev_attr_ep11_serialnr.attr, + &dev_attr_ep11_card_op_modes.attr, + NULL, +}; + +static const struct attribute_group ep11_card_attr_grp = { + .attrs = ep11_card_attrs, +}; + +/* + * EP11 queue additional device attributes + */ + +static ssize_t ep11_mkvps_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int n = 0; + struct ep11_domain_info di; + struct zcrypt_queue *zq = to_ap_queue(dev)->private; + static const char * const cwk_state[] = { "invalid", "valid" }; + static const char * const nwk_state[] = { "empty", "uncommitted", + "committed" }; + + memset(&di, 0, sizeof(di)); + + if (zq->online) + ep11_get_domain_info(AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + &di); + + if (di.cur_wk_state == '0') { + n = snprintf(buf, PAGE_SIZE, "WK CUR: %s -\n", + cwk_state[di.cur_wk_state - '0']); + } else if (di.cur_wk_state == '1') { + n = snprintf(buf, PAGE_SIZE, "WK CUR: %s 0x", + cwk_state[di.cur_wk_state - '0']); + bin2hex(buf + n, di.cur_wkvp, sizeof(di.cur_wkvp)); + n += 2 * sizeof(di.cur_wkvp); + n += snprintf(buf + n, PAGE_SIZE - n, "\n"); + } else + n = snprintf(buf, PAGE_SIZE, "WK CUR: - -\n"); + + if (di.new_wk_state == '0') { + n += snprintf(buf + n, PAGE_SIZE - n, "WK NEW: %s -\n", + nwk_state[di.new_wk_state - '0']); + } else if (di.new_wk_state >= '1' && di.new_wk_state <= '2') { + n += snprintf(buf + n, PAGE_SIZE - n, "WK NEW: %s 0x", + nwk_state[di.new_wk_state - '0']); + bin2hex(buf + n, di.new_wkvp, sizeof(di.new_wkvp)); + n += 2 * sizeof(di.new_wkvp); + n += snprintf(buf + n, PAGE_SIZE - n, "\n"); + } else + n += snprintf(buf + n, PAGE_SIZE - n, "WK NEW: - -\n"); + + return n; +} + +static struct device_attribute dev_attr_ep11_mkvps = + __ATTR(mkvps, 0444, ep11_mkvps_show, NULL); + +static ssize_t ep11_queue_op_modes_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int i, n = 0; + struct ep11_domain_info di; + struct zcrypt_queue *zq = to_ap_queue(dev)->private; + + memset(&di, 0, sizeof(di)); + + if (zq->online) + ep11_get_domain_info(AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), + &di); + + for (i = 0; ep11_op_modes[i].mode_txt; i++) { + if (di.op_mode & (1 << ep11_op_modes[i].mode_bit)) { + if (n > 0) + buf[n++] = ' '; + n += snprintf(buf + n, PAGE_SIZE - n, + "%s", ep11_op_modes[i].mode_txt); + } + } + n += snprintf(buf + n, PAGE_SIZE - n, "\n"); + + return n; +} + +static struct device_attribute dev_attr_ep11_queue_op_modes = + __ATTR(op_modes, 0444, ep11_queue_op_modes_show, NULL); + +static struct attribute *ep11_queue_attrs[] = { + &dev_attr_ep11_mkvps.attr, + &dev_attr_ep11_queue_op_modes.attr, + NULL, +}; + +static const struct attribute_group ep11_queue_attr_grp = { + .attrs = ep11_queue_attrs, +}; + /** * Probe function for CEX4/CEX5/CEX6/CEX7 card device. It always * accepts the AP device since the bus_match already checked @@ -313,7 +532,12 @@ static int zcrypt_cex4_card_probe(struct ap_device *ap_dev) if (ap_test_bit(&ac->functions, AP_FUNC_COPRO)) { rc = sysfs_create_group(&ap_dev->device.kobj, - &cca_card_attr_group); + &cca_card_attr_grp); + if (rc) + zcrypt_card_unregister(zc); + } else if (ap_test_bit(&ac->functions, AP_FUNC_EP11)) { + rc = sysfs_create_group(&ap_dev->device.kobj, + &ep11_card_attr_grp); if (rc) zcrypt_card_unregister(zc); } @@ -332,7 +556,9 @@ static void zcrypt_cex4_card_remove(struct ap_device *ap_dev) struct zcrypt_card *zc = ac->private; if (ap_test_bit(&ac->functions, AP_FUNC_COPRO)) - sysfs_remove_group(&ap_dev->device.kobj, &cca_card_attr_group); + sysfs_remove_group(&ap_dev->device.kobj, &cca_card_attr_grp); + else if (ap_test_bit(&ac->functions, AP_FUNC_EP11)) + sysfs_remove_group(&ap_dev->device.kobj, &ep11_card_attr_grp); if (zc) zcrypt_card_unregister(zc); } @@ -394,7 +620,12 @@ static int zcrypt_cex4_queue_probe(struct ap_device *ap_dev) if (ap_test_bit(&aq->card->functions, AP_FUNC_COPRO)) { rc = sysfs_create_group(&ap_dev->device.kobj, - &cca_queue_attr_group); + &cca_queue_attr_grp); + if (rc) + zcrypt_queue_unregister(zq); + } else if (ap_test_bit(&aq->card->functions, AP_FUNC_EP11)) { + rc = sysfs_create_group(&ap_dev->device.kobj, + &ep11_queue_attr_grp); if (rc) zcrypt_queue_unregister(zq); } @@ -413,7 +644,9 @@ static void zcrypt_cex4_queue_remove(struct ap_device *ap_dev) struct zcrypt_queue *zq = aq->private; if (ap_test_bit(&aq->card->functions, AP_FUNC_COPRO)) - sysfs_remove_group(&ap_dev->device.kobj, &cca_queue_attr_group); + sysfs_remove_group(&ap_dev->device.kobj, &cca_queue_attr_grp); + else if (ap_test_bit(&aq->card->functions, AP_FUNC_EP11)) + sysfs_remove_group(&ap_dev->device.kobj, &ep11_queue_attr_grp); if (zq) zcrypt_queue_unregister(zq); } diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c new file mode 100644 index 000000000000..d4caf46ff9df --- /dev/null +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -0,0 +1,1293 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright IBM Corp. 2019 + * Author(s): Harald Freudenberger <freude@linux.ibm.com> + * + * Collection of EP11 misc functions used by zcrypt and pkey + */ + +#define KMSG_COMPONENT "zcrypt" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/random.h> +#include <asm/zcrypt.h> +#include <asm/pkey.h> + +#include "ap_bus.h" +#include "zcrypt_api.h" +#include "zcrypt_debug.h" +#include "zcrypt_msgtype6.h" +#include "zcrypt_ep11misc.h" +#include "zcrypt_ccamisc.h" + +#define DEBUG_DBG(...) ZCRYPT_DBF(DBF_DEBUG, ##__VA_ARGS__) +#define DEBUG_INFO(...) ZCRYPT_DBF(DBF_INFO, ##__VA_ARGS__) +#define DEBUG_WARN(...) ZCRYPT_DBF(DBF_WARN, ##__VA_ARGS__) +#define DEBUG_ERR(...) ZCRYPT_DBF(DBF_ERR, ##__VA_ARGS__) + +/* default iv used here */ +static const u8 def_iv[16] = { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }; + +/* ep11 card info cache */ +struct card_list_entry { + struct list_head list; + u16 cardnr; + struct ep11_card_info info; +}; +static LIST_HEAD(card_list); +static DEFINE_SPINLOCK(card_list_lock); + +static int card_cache_fetch(u16 cardnr, struct ep11_card_info *ci) +{ + int rc = -ENOENT; + struct card_list_entry *ptr; + + spin_lock_bh(&card_list_lock); + list_for_each_entry(ptr, &card_list, list) { + if (ptr->cardnr == cardnr) { + memcpy(ci, &ptr->info, sizeof(*ci)); + rc = 0; + break; + } + } + spin_unlock_bh(&card_list_lock); + + return rc; +} + +static void card_cache_update(u16 cardnr, const struct ep11_card_info *ci) +{ + int found = 0; + struct card_list_entry *ptr; + + spin_lock_bh(&card_list_lock); + list_for_each_entry(ptr, &card_list, list) { + if (ptr->cardnr == cardnr) { + memcpy(&ptr->info, ci, sizeof(*ci)); + found = 1; + break; + } + } + if (!found) { + ptr = kmalloc(sizeof(*ptr), GFP_ATOMIC); + if (!ptr) { + spin_unlock_bh(&card_list_lock); + return; + } + ptr->cardnr = cardnr; + memcpy(&ptr->info, ci, sizeof(*ci)); + list_add(&ptr->list, &card_list); + } + spin_unlock_bh(&card_list_lock); +} + +static void card_cache_scrub(u16 cardnr) +{ + struct card_list_entry *ptr; + + spin_lock_bh(&card_list_lock); + list_for_each_entry(ptr, &card_list, list) { + if (ptr->cardnr == cardnr) { + list_del(&ptr->list); + kfree(ptr); + break; + } + } + spin_unlock_bh(&card_list_lock); +} + +static void __exit card_cache_free(void) +{ + struct card_list_entry *ptr, *pnext; + + spin_lock_bh(&card_list_lock); + list_for_each_entry_safe(ptr, pnext, &card_list, list) { + list_del(&ptr->list); + kfree(ptr); + } + spin_unlock_bh(&card_list_lock); +} + +/* + * Simple check if the key blob is a valid EP11 secure AES key. + */ +int ep11_check_aeskeyblob(debug_info_t *dbg, int dbflvl, + const u8 *key, int keybitsize, + int checkcpacfexport) +{ + struct ep11keyblob *kb = (struct ep11keyblob *) key; + +#define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) + + if (kb->head.type != TOKTYPE_NON_CCA) { + if (dbg) + DBF("%s key check failed, type 0x%02x != 0x%02x\n", + __func__, (int) kb->head.type, TOKTYPE_NON_CCA); + return -EINVAL; + } + if (kb->head.version != TOKVER_EP11_AES) { + if (dbg) + DBF("%s key check failed, version 0x%02x != 0x%02x\n", + __func__, (int) kb->head.version, TOKVER_EP11_AES); + return -EINVAL; + } + if (kb->version != EP11_STRUCT_MAGIC) { + if (dbg) + DBF("%s key check failed, magic 0x%04x != 0x%04x\n", + __func__, (int) kb->version, EP11_STRUCT_MAGIC); + return -EINVAL; + } + switch (kb->head.keybitlen) { + case 128: + case 192: + case 256: + break; + default: + if (dbg) + DBF("%s key check failed, keybitlen %d invalid\n", + __func__, (int) kb->head.keybitlen); + return -EINVAL; + } + if (keybitsize > 0 && keybitsize != (int) kb->head.keybitlen) { + DBF("%s key check failed, keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + if (checkcpacfexport && !(kb->attr & EP11_BLOB_PKEY_EXTRACTABLE)) { + if (dbg) + DBF("%s key check failed, PKEY_EXTRACTABLE is 0\n", + __func__); + return -EINVAL; + } +#undef DBF + + return 0; +} +EXPORT_SYMBOL(ep11_check_aeskeyblob); + +/* + * Helper function which calls zcrypt_send_ep11_cprb with + * memory management segment adjusted to kernel space + * so that the copy_from_user called within this + * function do in fact copy from kernel space. + */ +static inline int _zcrypt_send_ep11_cprb(struct ep11_urb *urb) +{ + int rc; + mm_segment_t old_fs = get_fs(); + + set_fs(KERNEL_DS); + rc = zcrypt_send_ep11_cprb(urb); + set_fs(old_fs); + + return rc; +} + +/* + * Allocate and prepare ep11 cprb plus additional payload. + */ +static inline struct ep11_cprb *alloc_cprb(size_t payload_len) +{ + size_t len = sizeof(struct ep11_cprb) + payload_len; + struct ep11_cprb *cprb; + + cprb = kmalloc(len, GFP_KERNEL); + if (!cprb) + return NULL; + + memset(cprb, 0, len); + cprb->cprb_len = sizeof(struct ep11_cprb); + cprb->cprb_ver_id = 0x04; + memcpy(cprb->func_id, "T4", 2); + cprb->ret_code = 0xFFFFFFFF; + cprb->payload_len = payload_len; + + return cprb; +} + +/* + * Some helper functions related to ASN1 encoding. + * Limited to length info <= 2 byte. + */ + +#define ASN1TAGLEN(x) (2 + (x) + ((x) > 127 ? 1 : 0) + ((x) > 255 ? 1 : 0)) + +static int asn1tag_write(u8 *ptr, u8 tag, const u8 *pvalue, u16 valuelen) +{ + ptr[0] = tag; + if (valuelen > 255) { + ptr[1] = 0x82; + *((u16 *)(ptr + 2)) = valuelen; + memcpy(ptr + 4, pvalue, valuelen); + return 4 + valuelen; + } + if (valuelen > 127) { + ptr[1] = 0x81; + ptr[2] = (u8) valuelen; + memcpy(ptr + 3, pvalue, valuelen); + return 3 + valuelen; + } + ptr[1] = (u8) valuelen; + memcpy(ptr + 2, pvalue, valuelen); + return 2 + valuelen; +} + +/* EP11 payload > 127 bytes starts with this struct */ +struct pl_head { + u8 tag; + u8 lenfmt; + u16 len; + u8 func_tag; + u8 func_len; + u32 func; + u8 dom_tag; + u8 dom_len; + u32 dom; +} __packed; + +/* prep ep11 payload head helper function */ +static inline void prep_head(struct pl_head *h, + size_t pl_size, int api, int func) +{ + h->tag = 0x30; + h->lenfmt = 0x82; + h->len = pl_size - 4; + h->func_tag = 0x04; + h->func_len = sizeof(u32); + h->func = (api << 16) + func; + h->dom_tag = 0x04; + h->dom_len = sizeof(u32); +} + +/* prep urb helper function */ +static inline void prep_urb(struct ep11_urb *u, + struct ep11_target_dev *t, int nt, + struct ep11_cprb *req, size_t req_len, + struct ep11_cprb *rep, size_t rep_len) +{ + u->targets = (u8 __user *) t; + u->targets_num = nt; + u->req = (u8 __user *) req; + u->req_len = req_len; + u->resp = (u8 __user *) rep; + u->resp_len = rep_len; +} + +/* Check ep11 reply payload, return 0 or suggested errno value. */ +static int check_reply_pl(const u8 *pl, const char *func) +{ + int len; + u32 ret; + + /* start tag */ + if (*pl++ != 0x30) { + DEBUG_ERR("%s reply start tag mismatch\n", func); + return -EIO; + } + + /* payload length format */ + if (*pl < 127) { + len = *pl; + pl++; + } else if (*pl == 0x81) { + pl++; + len = *pl; + pl++; + } else if (*pl == 0x82) { + pl++; + len = *((u16 *)pl); + pl += 2; + } else { + DEBUG_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n", + func, *pl); + return -EIO; + } + + /* len should cover at least 3 fields with 32 bit value each */ + if (len < 3 * 6) { + DEBUG_ERR("%s reply length %d too small\n", func, len); + return -EIO; + } + + /* function tag, length and value */ + if (pl[0] != 0x04 || pl[1] != 0x04) { + DEBUG_ERR("%s function tag or length mismatch\n", func); + return -EIO; + } + pl += 6; + + /* dom tag, length and value */ + if (pl[0] != 0x04 || pl[1] != 0x04) { + DEBUG_ERR("%s dom tag or length mismatch\n", func); + return -EIO; + } + pl += 6; + + /* return value tag, length and value */ + if (pl[0] != 0x04 || pl[1] != 0x04) { + DEBUG_ERR("%s return value tag or length mismatch\n", func); + return -EIO; + } + pl += 2; + ret = *((u32 *)pl); + if (ret != 0) { + DEBUG_ERR("%s return value 0x%04x != 0\n", func, ret); + return -EIO; + } + + return 0; +} + + +/* + * Helper function which does an ep11 query with given query type. + */ +static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, + size_t buflen, u8 *buf) +{ + struct ep11_info_req_pl { + struct pl_head head; + u8 query_type_tag; + u8 query_type_len; + u32 query_type; + u8 query_subtype_tag; + u8 query_subtype_len; + u32 query_subtype; + } __packed * req_pl; + struct ep11_info_rep_pl { + struct pl_head head; + u8 rc_tag; + u8 rc_len; + u32 rc; + u8 data_tag; + u8 data_lenfmt; + u16 data_len; + } __packed * rep_pl; + struct ep11_cprb *req = NULL, *rep = NULL; + struct ep11_target_dev target; + struct ep11_urb *urb = NULL; + int api = 1, rc = -ENOMEM; + + /* request cprb and payload */ + req = alloc_cprb(sizeof(struct ep11_info_req_pl)); + if (!req) + goto out; + req_pl = (struct ep11_info_req_pl *) (((u8 *) req) + sizeof(*req)); + prep_head(&req_pl->head, sizeof(*req_pl), api, 38); /* get xcp info */ + req_pl->query_type_tag = 0x04; + req_pl->query_type_len = sizeof(u32); + req_pl->query_type = query_type; + req_pl->query_subtype_tag = 0x04; + req_pl->query_subtype_len = sizeof(u32); + + /* reply cprb and payload */ + rep = alloc_cprb(sizeof(struct ep11_info_rep_pl) + buflen); + if (!rep) + goto out; + rep_pl = (struct ep11_info_rep_pl *) (((u8 *) rep) + sizeof(*rep)); + + /* urb and target */ + urb = kmalloc(sizeof(struct ep11_urb), GFP_KERNEL); + if (!urb) + goto out; + target.ap_id = cardnr; + target.dom_id = domain; + prep_urb(urb, &target, 1, + req, sizeof(*req) + sizeof(*req_pl), + rep, sizeof(*rep) + sizeof(*rep_pl) + buflen); + + rc = _zcrypt_send_ep11_cprb(urb); + if (rc) { + DEBUG_ERR( + "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int) cardnr, (int) domain, rc); + goto out; + } + + rc = check_reply_pl((u8 *)rep_pl, __func__); + if (rc) + goto out; + if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { + DEBUG_ERR("%s unknown reply data format\n", __func__); + rc = -EIO; + goto out; + } + if (rep_pl->data_len > buflen) { + DEBUG_ERR("%s mismatch between reply data len and buffer len\n", + __func__); + rc = -ENOSPC; + goto out; + } + + memcpy(buf, ((u8 *) rep_pl) + sizeof(*rep_pl), rep_pl->data_len); + +out: + kfree(req); + kfree(rep); + kfree(urb); + return rc; +} + +/* + * Provide information about an EP11 card. + */ +int ep11_get_card_info(u16 card, struct ep11_card_info *info, int verify) +{ + int rc; + struct ep11_module_query_info { + u32 API_ord_nr; + u32 firmware_id; + u8 FW_major_vers; + u8 FW_minor_vers; + u8 CSP_major_vers; + u8 CSP_minor_vers; + u8 fwid[32]; + u8 xcp_config_hash[32]; + u8 CSP_config_hash[32]; + u8 serial[16]; + u8 module_date_time[16]; + u64 op_mode; + u32 PKCS11_flags; + u32 ext_flags; + u32 domains; + u32 sym_state_bytes; + u32 digest_state_bytes; + u32 pin_blob_bytes; + u32 SPKI_bytes; + u32 priv_key_blob_bytes; + u32 sym_blob_bytes; + u32 max_payload_bytes; + u32 CP_profile_bytes; + u32 max_CP_index; + } __packed * pmqi = NULL; + + rc = card_cache_fetch(card, info); + if (rc || verify) { + pmqi = kmalloc(sizeof(*pmqi), GFP_KERNEL); + if (!pmqi) + return -ENOMEM; + rc = ep11_query_info(card, AUTOSEL_DOM, + 0x01 /* module info query */, + sizeof(*pmqi), (u8 *) pmqi); + if (rc) { + if (rc == -ENODEV) + card_cache_scrub(card); + goto out; + } + memset(info, 0, sizeof(*info)); + info->API_ord_nr = pmqi->API_ord_nr; + info->FW_version = + (pmqi->FW_major_vers << 8) + pmqi->FW_minor_vers; + memcpy(info->serial, pmqi->serial, sizeof(info->serial)); + info->op_mode = pmqi->op_mode; + card_cache_update(card, info); + } + +out: + kfree(pmqi); + return rc; +} +EXPORT_SYMBOL(ep11_get_card_info); + +/* + * Provide information about a domain within an EP11 card. + */ +int ep11_get_domain_info(u16 card, u16 domain, struct ep11_domain_info *info) +{ + int rc; + struct ep11_domain_query_info { + u32 dom_index; + u8 cur_WK_VP[32]; + u8 new_WK_VP[32]; + u32 dom_flags; + u64 op_mode; + } __packed * p_dom_info; + + p_dom_info = kmalloc(sizeof(*p_dom_info), GFP_KERNEL); + if (!p_dom_info) + return -ENOMEM; + + rc = ep11_query_info(card, domain, 0x03 /* domain info query */, + sizeof(*p_dom_info), (u8 *) p_dom_info); + if (rc) + goto out; + + memset(info, 0, sizeof(*info)); + info->cur_wk_state = '0'; + info->new_wk_state = '0'; + if (p_dom_info->dom_flags & 0x10 /* left imprint mode */) { + if (p_dom_info->dom_flags & 0x02 /* cur wk valid */) { + info->cur_wk_state = '1'; + memcpy(info->cur_wkvp, p_dom_info->cur_WK_VP, 32); + } + if (p_dom_info->dom_flags & 0x04 /* new wk present */ + || p_dom_info->dom_flags & 0x08 /* new wk committed */) { + info->new_wk_state = + p_dom_info->dom_flags & 0x08 ? '2' : '1'; + memcpy(info->new_wkvp, p_dom_info->new_WK_VP, 32); + } + } + info->op_mode = p_dom_info->op_mode; + +out: + kfree(p_dom_info); + return rc; +} +EXPORT_SYMBOL(ep11_get_domain_info); + +/* + * Default EP11 AES key generate attributes, used when no keygenflags given: + * XCP_BLOB_ENCRYPT | XCP_BLOB_DECRYPT | XCP_BLOB_PROTKEY_EXTRACTABLE + */ +#define KEY_ATTR_DEFAULTS 0x00200c00 + +int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, + u8 *keybuf, size_t *keybufsize) +{ + struct keygen_req_pl { + struct pl_head head; + u8 var_tag; + u8 var_len; + u32 var; + u8 keybytes_tag; + u8 keybytes_len; + u32 keybytes; + u8 mech_tag; + u8 mech_len; + u32 mech; + u8 attr_tag; + u8 attr_len; + u32 attr_header; + u32 attr_bool_mask; + u32 attr_bool_bits; + u32 attr_val_len_type; + u32 attr_val_len_value; + u8 pin_tag; + u8 pin_len; + } __packed * req_pl; + struct keygen_rep_pl { + struct pl_head head; + u8 rc_tag; + u8 rc_len; + u32 rc; + u8 data_tag; + u8 data_lenfmt; + u16 data_len; + u8 data[512]; + } __packed * rep_pl; + struct ep11_cprb *req = NULL, *rep = NULL; + struct ep11_target_dev target; + struct ep11_urb *urb = NULL; + struct ep11keyblob *kb; + int api, rc = -ENOMEM; + + switch (keybitsize) { + case 128: + case 192: + case 256: + break; + default: + DEBUG_ERR( + "%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + rc = -EINVAL; + goto out; + } + + /* request cprb and payload */ + req = alloc_cprb(sizeof(struct keygen_req_pl)); + if (!req) + goto out; + req_pl = (struct keygen_req_pl *) (((u8 *) req) + sizeof(*req)); + api = (!keygenflags || keygenflags & 0x00200000) ? 4 : 1; + prep_head(&req_pl->head, sizeof(*req_pl), api, 21); /* GenerateKey */ + req_pl->var_tag = 0x04; + req_pl->var_len = sizeof(u32); + req_pl->keybytes_tag = 0x04; + req_pl->keybytes_len = sizeof(u32); + req_pl->keybytes = keybitsize / 8; + req_pl->mech_tag = 0x04; + req_pl->mech_len = sizeof(u32); + req_pl->mech = 0x00001080; /* CKM_AES_KEY_GEN */ + req_pl->attr_tag = 0x04; + req_pl->attr_len = 5 * sizeof(u32); + req_pl->attr_header = 0x10010000; + req_pl->attr_bool_mask = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS; + req_pl->attr_bool_bits = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS; + req_pl->attr_val_len_type = 0x00000161; /* CKA_VALUE_LEN */ + req_pl->attr_val_len_value = keybitsize / 8; + req_pl->pin_tag = 0x04; + + /* reply cprb and payload */ + rep = alloc_cprb(sizeof(struct keygen_rep_pl)); + if (!rep) + goto out; + rep_pl = (struct keygen_rep_pl *) (((u8 *) rep) + sizeof(*rep)); + + /* urb and target */ + urb = kmalloc(sizeof(struct ep11_urb), GFP_KERNEL); + if (!urb) + goto out; + target.ap_id = card; + target.dom_id = domain; + prep_urb(urb, &target, 1, + req, sizeof(*req) + sizeof(*req_pl), + rep, sizeof(*rep) + sizeof(*rep_pl)); + + rc = _zcrypt_send_ep11_cprb(urb); + if (rc) { + DEBUG_ERR( + "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int) card, (int) domain, rc); + goto out; + } + + rc = check_reply_pl((u8 *)rep_pl, __func__); + if (rc) + goto out; + if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { + DEBUG_ERR("%s unknown reply data format\n", __func__); + rc = -EIO; + goto out; + } + if (rep_pl->data_len > *keybufsize) { + DEBUG_ERR("%s mismatch reply data len / key buffer len\n", + __func__); + rc = -ENOSPC; + goto out; + } + + /* copy key blob and set header values */ + memcpy(keybuf, rep_pl->data, rep_pl->data_len); + *keybufsize = rep_pl->data_len; + kb = (struct ep11keyblob *) keybuf; + kb->head.type = TOKTYPE_NON_CCA; + kb->head.len = rep_pl->data_len; + kb->head.version = TOKVER_EP11_AES; + kb->head.keybitlen = keybitsize; + +out: + kfree(req); + kfree(rep); + kfree(urb); + return rc; +} +EXPORT_SYMBOL(ep11_genaeskey); + +static int ep11_cryptsingle(u16 card, u16 domain, + u16 mode, u32 mech, const u8 *iv, + const u8 *key, size_t keysize, + const u8 *inbuf, size_t inbufsize, + u8 *outbuf, size_t *outbufsize) +{ + struct crypt_req_pl { + struct pl_head head; + u8 var_tag; + u8 var_len; + u32 var; + u8 mech_tag; + u8 mech_len; + u32 mech; + /* + * maybe followed by iv data + * followed by key tag + key blob + * followed by plaintext tag + plaintext + */ + } __packed * req_pl; + struct crypt_rep_pl { + struct pl_head head; + u8 rc_tag; + u8 rc_len; + u32 rc; + u8 data_tag; + u8 data_lenfmt; + /* data follows */ + } __packed * rep_pl; + struct ep11_cprb *req = NULL, *rep = NULL; + struct ep11_target_dev target; + struct ep11_urb *urb = NULL; + size_t req_pl_size, rep_pl_size; + int n, api = 1, rc = -ENOMEM; + u8 *p; + + /* the simple asn1 coding used has length limits */ + if (keysize > 0xFFFF || inbufsize > 0xFFFF) + return -EINVAL; + + /* request cprb and payload */ + req_pl_size = sizeof(struct crypt_req_pl) + (iv ? 16 : 0) + + ASN1TAGLEN(keysize) + ASN1TAGLEN(inbufsize); + req = alloc_cprb(req_pl_size); + if (!req) + goto out; + req_pl = (struct crypt_req_pl *) (((u8 *) req) + sizeof(*req)); + prep_head(&req_pl->head, req_pl_size, api, (mode ? 20 : 19)); + req_pl->var_tag = 0x04; + req_pl->var_len = sizeof(u32); + /* mech is mech + mech params (iv here) */ + req_pl->mech_tag = 0x04; + req_pl->mech_len = sizeof(u32) + (iv ? 16 : 0); + req_pl->mech = (mech ? mech : 0x00001085); /* CKM_AES_CBC_PAD */ + p = ((u8 *) req_pl) + sizeof(*req_pl); + if (iv) { + memcpy(p, iv, 16); + p += 16; + } + /* key and input data */ + p += asn1tag_write(p, 0x04, key, keysize); + p += asn1tag_write(p, 0x04, inbuf, inbufsize); + + /* reply cprb and payload, assume out data size <= in data size + 32 */ + rep_pl_size = sizeof(struct crypt_rep_pl) + ASN1TAGLEN(inbufsize + 32); + rep = alloc_cprb(rep_pl_size); + if (!rep) + goto out; + rep_pl = (struct crypt_rep_pl *) (((u8 *) rep) + sizeof(*rep)); + + /* urb and target */ + urb = kmalloc(sizeof(struct ep11_urb), GFP_KERNEL); + if (!urb) + goto out; + target.ap_id = card; + target.dom_id = domain; + prep_urb(urb, &target, 1, + req, sizeof(*req) + req_pl_size, + rep, sizeof(*rep) + rep_pl_size); + + rc = _zcrypt_send_ep11_cprb(urb); + if (rc) { + DEBUG_ERR( + "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int) card, (int) domain, rc); + goto out; + } + + rc = check_reply_pl((u8 *)rep_pl, __func__); + if (rc) + goto out; + if (rep_pl->data_tag != 0x04) { + DEBUG_ERR("%s unknown reply data format\n", __func__); + rc = -EIO; + goto out; + } + p = ((u8 *) rep_pl) + sizeof(*rep_pl); + if (rep_pl->data_lenfmt <= 127) + n = rep_pl->data_lenfmt; + else if (rep_pl->data_lenfmt == 0x81) + n = *p++; + else if (rep_pl->data_lenfmt == 0x82) { + n = *((u16 *) p); + p += 2; + } else { + DEBUG_ERR("%s unknown reply data length format 0x%02hhx\n", + __func__, rep_pl->data_lenfmt); + rc = -EIO; + goto out; + } + if (n > *outbufsize) { + DEBUG_ERR("%s mismatch reply data len %d / output buffer %zu\n", + __func__, n, *outbufsize); + rc = -ENOSPC; + goto out; + } + + memcpy(outbuf, p, n); + *outbufsize = n; + +out: + kfree(req); + kfree(rep); + kfree(urb); + return rc; +} + +static int ep11_unwrapkey(u16 card, u16 domain, + const u8 *kek, size_t keksize, + const u8 *enckey, size_t enckeysize, + u32 mech, const u8 *iv, + u32 keybitsize, u32 keygenflags, + u8 *keybuf, size_t *keybufsize) +{ + struct uw_req_pl { + struct pl_head head; + u8 attr_tag; + u8 attr_len; + u32 attr_header; + u32 attr_bool_mask; + u32 attr_bool_bits; + u32 attr_key_type; + u32 attr_key_type_value; + u32 attr_val_len; + u32 attr_val_len_value; + u8 mech_tag; + u8 mech_len; + u32 mech; + /* + * maybe followed by iv data + * followed by kek tag + kek blob + * followed by empty mac tag + * followed by empty pin tag + * followed by encryted key tag + bytes + */ + } __packed * req_pl; + struct uw_rep_pl { + struct pl_head head; + u8 rc_tag; + u8 rc_len; + u32 rc; + u8 data_tag; + u8 data_lenfmt; + u16 data_len; + u8 data[512]; + } __packed * rep_pl; + struct ep11_cprb *req = NULL, *rep = NULL; + struct ep11_target_dev target; + struct ep11_urb *urb = NULL; + struct ep11keyblob *kb; + size_t req_pl_size; + int api, rc = -ENOMEM; + u8 *p; + + /* request cprb and payload */ + req_pl_size = sizeof(struct uw_req_pl) + (iv ? 16 : 0) + + ASN1TAGLEN(keksize) + 4 + ASN1TAGLEN(enckeysize); + req = alloc_cprb(req_pl_size); + if (!req) + goto out; + req_pl = (struct uw_req_pl *) (((u8 *) req) + sizeof(*req)); + api = (!keygenflags || keygenflags & 0x00200000) ? 4 : 1; + prep_head(&req_pl->head, req_pl_size, api, 34); /* UnwrapKey */ + req_pl->attr_tag = 0x04; + req_pl->attr_len = 7 * sizeof(u32); + req_pl->attr_header = 0x10020000; + req_pl->attr_bool_mask = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS; + req_pl->attr_bool_bits = keygenflags ? keygenflags : KEY_ATTR_DEFAULTS; + req_pl->attr_key_type = 0x00000100; /* CKA_KEY_TYPE */ + req_pl->attr_key_type_value = 0x0000001f; /* CKK_AES */ + req_pl->attr_val_len = 0x00000161; /* CKA_VALUE_LEN */ + req_pl->attr_val_len_value = keybitsize / 8; + /* mech is mech + mech params (iv here) */ + req_pl->mech_tag = 0x04; + req_pl->mech_len = sizeof(u32) + (iv ? 16 : 0); + req_pl->mech = (mech ? mech : 0x00001085); /* CKM_AES_CBC_PAD */ + p = ((u8 *) req_pl) + sizeof(*req_pl); + if (iv) { + memcpy(p, iv, 16); + p += 16; + } + /* kek */ + p += asn1tag_write(p, 0x04, kek, keksize); + /* empty mac key tag */ + *p++ = 0x04; + *p++ = 0; + /* empty pin tag */ + *p++ = 0x04; + *p++ = 0; + /* encrytped key value tag and bytes */ + p += asn1tag_write(p, 0x04, enckey, enckeysize); + + /* reply cprb and payload */ + rep = alloc_cprb(sizeof(struct uw_rep_pl)); + if (!rep) + goto out; + rep_pl = (struct uw_rep_pl *) (((u8 *) rep) + sizeof(*rep)); + + /* urb and target */ + urb = kmalloc(sizeof(struct ep11_urb), GFP_KERNEL); + if (!urb) + goto out; + target.ap_id = card; + target.dom_id = domain; + prep_urb(urb, &target, 1, + req, sizeof(*req) + req_pl_size, + rep, sizeof(*rep) + sizeof(*rep_pl)); + + rc = _zcrypt_send_ep11_cprb(urb); + if (rc) { + DEBUG_ERR( + "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int) card, (int) domain, rc); + goto out; + } + + rc = check_reply_pl((u8 *)rep_pl, __func__); + if (rc) + goto out; + if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { + DEBUG_ERR("%s unknown reply data format\n", __func__); + rc = -EIO; + goto out; + } + if (rep_pl->data_len > *keybufsize) { + DEBUG_ERR("%s mismatch reply data len / key buffer len\n", + __func__); + rc = -ENOSPC; + goto out; + } + + /* copy key blob and set header values */ + memcpy(keybuf, rep_pl->data, rep_pl->data_len); + *keybufsize = rep_pl->data_len; + kb = (struct ep11keyblob *) keybuf; + kb->head.type = TOKTYPE_NON_CCA; + kb->head.len = rep_pl->data_len; + kb->head.version = TOKVER_EP11_AES; + kb->head.keybitlen = keybitsize; + +out: + kfree(req); + kfree(rep); + kfree(urb); + return rc; +} + +static int ep11_wrapkey(u16 card, u16 domain, + const u8 *key, size_t keysize, + u32 mech, const u8 *iv, + u8 *databuf, size_t *datasize) +{ + struct wk_req_pl { + struct pl_head head; + u8 var_tag; + u8 var_len; + u32 var; + u8 mech_tag; + u8 mech_len; + u32 mech; + /* + * followed by iv data + * followed by key tag + key blob + * followed by dummy kek param + * followed by dummy mac param + */ + } __packed * req_pl; + struct wk_rep_pl { + struct pl_head head; + u8 rc_tag; + u8 rc_len; + u32 rc; + u8 data_tag; + u8 data_lenfmt; + u16 data_len; + u8 data[512]; + } __packed * rep_pl; + struct ep11_cprb *req = NULL, *rep = NULL; + struct ep11_target_dev target; + struct ep11_urb *urb = NULL; + struct ep11keyblob *kb; + size_t req_pl_size; + int api, rc = -ENOMEM; + u8 *p; + + /* request cprb and payload */ + req_pl_size = sizeof(struct wk_req_pl) + (iv ? 16 : 0) + + ASN1TAGLEN(keysize) + 4; + req = alloc_cprb(req_pl_size); + if (!req) + goto out; + if (!mech || mech == 0x80060001) + req->flags |= 0x20; /* CPACF_WRAP needs special bit */ + req_pl = (struct wk_req_pl *) (((u8 *) req) + sizeof(*req)); + api = (!mech || mech == 0x80060001) ? 4 : 1; /* CKM_IBM_CPACF_WRAP */ + prep_head(&req_pl->head, req_pl_size, api, 33); /* WrapKey */ + req_pl->var_tag = 0x04; + req_pl->var_len = sizeof(u32); + /* mech is mech + mech params (iv here) */ + req_pl->mech_tag = 0x04; + req_pl->mech_len = sizeof(u32) + (iv ? 16 : 0); + req_pl->mech = (mech ? mech : 0x80060001); /* CKM_IBM_CPACF_WRAP */ + p = ((u8 *) req_pl) + sizeof(*req_pl); + if (iv) { + memcpy(p, iv, 16); + p += 16; + } + /* key blob */ + p += asn1tag_write(p, 0x04, key, keysize); + /* maybe the key argument needs the head data cleaned out */ + kb = (struct ep11keyblob *)(p - keysize); + if (kb->head.version == TOKVER_EP11_AES) + memset(&kb->head, 0, sizeof(kb->head)); + /* empty kek tag */ + *p++ = 0x04; + *p++ = 0; + /* empty mac tag */ + *p++ = 0x04; + *p++ = 0; + + /* reply cprb and payload */ + rep = alloc_cprb(sizeof(struct wk_rep_pl)); + if (!rep) + goto out; + rep_pl = (struct wk_rep_pl *) (((u8 *) rep) + sizeof(*rep)); + + /* urb and target */ + urb = kmalloc(sizeof(struct ep11_urb), GFP_KERNEL); + if (!urb) + goto out; + target.ap_id = card; + target.dom_id = domain; + prep_urb(urb, &target, 1, + req, sizeof(*req) + req_pl_size, + rep, sizeof(*rep) + sizeof(*rep_pl)); + + rc = _zcrypt_send_ep11_cprb(urb); + if (rc) { + DEBUG_ERR( + "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int) card, (int) domain, rc); + goto out; + } + + rc = check_reply_pl((u8 *)rep_pl, __func__); + if (rc) + goto out; + if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { + DEBUG_ERR("%s unknown reply data format\n", __func__); + rc = -EIO; + goto out; + } + if (rep_pl->data_len > *datasize) { + DEBUG_ERR("%s mismatch reply data len / data buffer len\n", + __func__); + rc = -ENOSPC; + goto out; + } + + /* copy the data from the cprb to the data buffer */ + memcpy(databuf, rep_pl->data, rep_pl->data_len); + *datasize = rep_pl->data_len; + +out: + kfree(req); + kfree(rep); + kfree(urb); + return rc; +} + +int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, + const u8 *clrkey, u8 *keybuf, size_t *keybufsize) +{ + int rc; + struct ep11keyblob *kb; + u8 encbuf[64], *kek = NULL; + size_t clrkeylen, keklen, encbuflen = sizeof(encbuf); + + if (keybitsize == 128 || keybitsize == 192 || keybitsize == 256) + clrkeylen = keybitsize / 8; + else { + DEBUG_ERR( + "%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); + return -EINVAL; + } + + /* allocate memory for the temp kek */ + keklen = MAXEP11AESKEYBLOBSIZE; + kek = kmalloc(keklen, GFP_ATOMIC); + if (!kek) { + rc = -ENOMEM; + goto out; + } + + /* Step 1: generate AES 256 bit random kek key */ + rc = ep11_genaeskey(card, domain, 256, + 0x00006c00, /* EN/DECRYTP, WRAP/UNWRAP */ + kek, &keklen); + if (rc) { + DEBUG_ERR( + "%s generate kek key failed, rc=%d\n", + __func__, rc); + goto out; + } + kb = (struct ep11keyblob *) kek; + memset(&kb->head, 0, sizeof(kb->head)); + + /* Step 2: encrypt clear key value with the kek key */ + rc = ep11_cryptsingle(card, domain, 0, 0, def_iv, kek, keklen, + clrkey, clrkeylen, encbuf, &encbuflen); + if (rc) { + DEBUG_ERR( + "%s encrypting key value with kek key failed, rc=%d\n", + __func__, rc); + goto out; + } + + /* Step 3: import the encrypted key value as a new key */ + rc = ep11_unwrapkey(card, domain, kek, keklen, + encbuf, encbuflen, 0, def_iv, + keybitsize, 0, keybuf, keybufsize); + if (rc) { + DEBUG_ERR( + "%s importing key value as new key failed,, rc=%d\n", + __func__, rc); + goto out; + } + +out: + kfree(kek); + return rc; +} +EXPORT_SYMBOL(ep11_clr2keyblob); + +int ep11_key2protkey(u16 card, u16 dom, const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype) +{ + int rc = -EIO; + u8 *wkbuf = NULL; + size_t wkbuflen = 256; + struct wk_info { + u16 version; + u8 res1[16]; + u32 pkeytype; + u32 pkeybitsize; + u64 pkeysize; + u8 res2[8]; + u8 pkey[0]; + } __packed * wki; + + /* alloc temp working buffer */ + wkbuf = kmalloc(wkbuflen, GFP_ATOMIC); + if (!wkbuf) + return -ENOMEM; + + /* ep11 secure key -> protected key + info */ + rc = ep11_wrapkey(card, dom, key, keylen, + 0, def_iv, wkbuf, &wkbuflen); + if (rc) { + DEBUG_ERR( + "%s rewrapping ep11 key to pkey failed, rc=%d\n", + __func__, rc); + goto out; + } + wki = (struct wk_info *) wkbuf; + + /* check struct version and pkey type */ + if (wki->version != 1 || wki->pkeytype != 1) { + DEBUG_ERR("%s wk info version %d or pkeytype %d mismatch.\n", + __func__, (int) wki->version, (int) wki->pkeytype); + rc = -EIO; + goto out; + } + + /* copy the tanslated protected key */ + switch (wki->pkeysize) { + case 16+32: + /* AES 128 protected key */ + if (protkeytype) + *protkeytype = PKEY_KEYTYPE_AES_128; + break; + case 24+32: + /* AES 192 protected key */ + if (protkeytype) + *protkeytype = PKEY_KEYTYPE_AES_192; + break; + case 32+32: + /* AES 256 protected key */ + if (protkeytype) + *protkeytype = PKEY_KEYTYPE_AES_256; + break; + default: + DEBUG_ERR("%s unknown/unsupported pkeysize %d\n", + __func__, (int) wki->pkeysize); + rc = -EIO; + goto out; + } + memcpy(protkey, wki->pkey, wki->pkeysize); + if (protkeylen) + *protkeylen = (u32) wki->pkeysize; + rc = 0; + +out: + kfree(wkbuf); + return rc; +} +EXPORT_SYMBOL(ep11_key2protkey); + +int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, + int minhwtype, int minapi, const u8 *wkvp) +{ + struct zcrypt_device_status_ext *device_status; + u32 *_apqns = NULL, _nr_apqns = 0; + int i, card, dom, rc = -ENOMEM; + struct ep11_domain_info edi; + struct ep11_card_info eci; + + /* fetch status of all crypto cards */ + device_status = kmalloc_array(MAX_ZDEV_ENTRIES_EXT, + sizeof(struct zcrypt_device_status_ext), + GFP_KERNEL); + if (!device_status) + return -ENOMEM; + zcrypt_device_status_mask_ext(device_status); + + /* allocate 1k space for up to 256 apqns */ + _apqns = kmalloc_array(256, sizeof(u32), GFP_KERNEL); + if (!_apqns) { + kfree(device_status); + return -ENOMEM; + } + + /* walk through all the crypto apqnss */ + for (i = 0; i < MAX_ZDEV_ENTRIES_EXT; i++) { + card = AP_QID_CARD(device_status[i].qid); + dom = AP_QID_QUEUE(device_status[i].qid); + /* check online state */ + if (!device_status[i].online) + continue; + /* check for ep11 functions */ + if (!(device_status[i].functions & 0x01)) + continue; + /* check cardnr */ + if (cardnr != 0xFFFF && card != cardnr) + continue; + /* check domain */ + if (domain != 0xFFFF && dom != domain) + continue; + /* check min hardware type */ + if (minhwtype && device_status[i].hwtype < minhwtype) + continue; + /* check min api version if given */ + if (minapi > 0) { + if (ep11_get_card_info(card, &eci, 0)) + continue; + if (minapi > eci.API_ord_nr) + continue; + } + /* check wkvp if given */ + if (wkvp) { + if (ep11_get_domain_info(card, dom, &edi)) + continue; + if (edi.cur_wk_state != '1') + continue; + if (memcmp(wkvp, edi.cur_wkvp, 16)) + continue; + } + /* apqn passed all filtering criterons, add to the array */ + if (_nr_apqns < 256) + _apqns[_nr_apqns++] = (((u16)card) << 16) | ((u16) dom); + } + + /* nothing found ? */ + if (!_nr_apqns) { + kfree(_apqns); + rc = -ENODEV; + } else { + /* no re-allocation, simple return the _apqns array */ + *apqns = _apqns; + *nr_apqns = _nr_apqns; + rc = 0; + } + + kfree(device_status); + return rc; +} +EXPORT_SYMBOL(ep11_findcard2); + +void __exit zcrypt_ep11misc_exit(void) +{ + card_cache_free(); +} diff --git a/drivers/s390/crypto/zcrypt_ep11misc.h b/drivers/s390/crypto/zcrypt_ep11misc.h new file mode 100644 index 000000000000..e3ed5ed1de86 --- /dev/null +++ b/drivers/s390/crypto/zcrypt_ep11misc.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright IBM Corp. 2019 + * Author(s): Harald Freudenberger <freude@linux.ibm.com> + * + * Collection of EP11 misc functions used by zcrypt and pkey + */ + +#ifndef _ZCRYPT_EP11MISC_H_ +#define _ZCRYPT_EP11MISC_H_ + +#include <asm/zcrypt.h> +#include <asm/pkey.h> + +#define TOKVER_EP11_AES 0x03 /* EP11 AES key blob */ + +#define EP11_API_V 4 /* highest known and supported EP11 API version */ + +#define EP11_STRUCT_MAGIC 0x1234 +#define EP11_BLOB_PKEY_EXTRACTABLE 0x200000 + +/* inside view of an EP11 secure key blob */ +struct ep11keyblob { + union { + u8 session[32]; + struct { + u8 type; /* 0x00 (TOKTYPE_NON_CCA) */ + u8 res0; /* unused */ + u16 len; /* total length in bytes of this blob */ + u8 version; /* 0x06 (TOKVER_EP11_AES) */ + u8 res1; /* unused */ + u16 keybitlen; /* clear key bit len, 0 for unknown */ + } head; + }; + u8 wkvp[16]; /* wrapping key verification pattern */ + u64 attr; /* boolean key attributes */ + u64 mode; /* mode bits */ + u16 version; /* 0x1234, EP11_STRUCT_MAGIC */ + u8 iv[14]; + u8 encrypted_key_data[144]; + u8 mac[32]; +} __packed; + +/* + * Simple check if the key blob is a valid EP11 secure AES key. + * If keybitsize is given, the bitsize of the key is also checked. + * If checkcpacfexport is enabled, the key is also checked for the + * attributes needed to export this key for CPACF use. + * Returns 0 on success or errno value on failure. + */ +int ep11_check_aeskeyblob(debug_info_t *dbg, int dbflvl, + const u8 *key, int keybitsize, + int checkcpacfexport); + +/* EP11 card info struct */ +struct ep11_card_info { + u32 API_ord_nr; /* API ordinal number */ + u16 FW_version; /* Firmware major and minor version */ + char serial[16]; /* serial number string (16 ascii, no 0x00 !) */ + u64 op_mode; /* card operational mode(s) */ +}; + +/* EP11 domain info struct */ +struct ep11_domain_info { + char cur_wk_state; /* '0' invalid, '1' valid */ + char new_wk_state; /* '0' empty, '1' uncommitted, '2' committed */ + u8 cur_wkvp[32]; /* current wrapping key verification pattern */ + u8 new_wkvp[32]; /* new wrapping key verification pattern */ + u64 op_mode; /* domain operational mode(s) */ +}; + +/* + * Provide information about an EP11 card. + */ +int ep11_get_card_info(u16 card, struct ep11_card_info *info, int verify); + +/* + * Provide information about a domain within an EP11 card. + */ +int ep11_get_domain_info(u16 card, u16 domain, struct ep11_domain_info *info); + +/* + * Generate (random) EP11 AES secure key. + */ +int ep11_genaeskey(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, + u8 *keybuf, size_t *keybufsize); + +/* + * Generate EP11 AES secure key with given clear key value. + */ +int ep11_clr2keyblob(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, + const u8 *clrkey, u8 *keybuf, size_t *keybufsize); + +/* + * Derive proteced key from EP11 AES secure key blob. + */ +int ep11_key2protkey(u16 cardnr, u16 domain, const u8 *key, size_t keylen, + u8 *protkey, u32 *protkeylen, u32 *protkeytype); + +/* + * Build a list of ep11 apqns meeting the following constrains: + * - apqn is online and is in fact an EP11 apqn + * - if cardnr is not FFFF only apqns with this cardnr + * - if domain is not FFFF only apqns with this domainnr + * - if minhwtype > 0 only apqns with hwtype >= minhwtype + * - if minapi > 0 only apqns with API_ord_nr >= minapi + * - if wkvp != NULL only apqns where the wkvp (EP11_WKVPLEN bytes) matches + * to the first EP11_WKVPLEN bytes of the wkvp of the current wrapping + * key for this domain. When a wkvp is given there will aways be a re-fetch + * of the domain info for the potential apqn - so this triggers an request + * reply to each apqn eligible. + * The array of apqn entries is allocated with kmalloc and returned in *apqns; + * the number of apqns stored into the list is returned in *nr_apqns. One apqn + * entry is simple a 32 bit value with 16 bit cardnr and 16 bit domain nr and + * may be casted to struct pkey_apqn. The return value is either 0 for success + * or a negative errno value. If no apqn meeting the criterias is found, + * -ENODEV is returned. + */ +int ep11_findcard2(u32 **apqns, u32 *nr_apqns, u16 cardnr, u16 domain, + int minhwtype, int minapi, const u8 *wkvp); + +void zcrypt_ep11misc_exit(void); + +#endif /* _ZCRYPT_EP11MISC_H_ */ diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c index 80c5a235d193..7b49e2e9fcde 100644 --- a/drivers/scsi/esas2r/esas2r_main.c +++ b/drivers/scsi/esas2r/esas2r_main.c @@ -617,6 +617,13 @@ static const struct file_operations esas2r_proc_fops = { .unlocked_ioctl = esas2r_proc_ioctl, }; +static const struct proc_ops esas2r_proc_ops = { + .proc_ioctl = esas2r_proc_ioctl, +#ifdef CONFIG_COMPAT + .proc_compat_ioctl = compat_ptr_ioctl, +#endif +}; + static struct Scsi_Host *esas2r_proc_host; static int esas2r_proc_major; @@ -728,7 +735,7 @@ const char *esas2r_info(struct Scsi_Host *sh) pde = proc_create(ATTONODE_NAME, 0, sh->hostt->proc_dir, - &esas2r_proc_fops); + &esas2r_proc_ops); if (!pde) { esas2r_log_dev(ESAS2R_LOG_WARN, diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c index df14597752ec..eed31021e788 100644 --- a/drivers/scsi/scsi_devinfo.c +++ b/drivers/scsi/scsi_devinfo.c @@ -736,13 +736,12 @@ out: return err; } -static const struct file_operations scsi_devinfo_proc_fops = { - .owner = THIS_MODULE, - .open = proc_scsi_devinfo_open, - .read = seq_read, - .write = proc_scsi_devinfo_write, - .llseek = seq_lseek, - .release = seq_release, +static const struct proc_ops scsi_devinfo_proc_ops = { + .proc_open = proc_scsi_devinfo_open, + .proc_read = seq_read, + .proc_write = proc_scsi_devinfo_write, + .proc_lseek = seq_lseek, + .proc_release = seq_release, }; #endif /* CONFIG_SCSI_PROC_FS */ @@ -867,7 +866,7 @@ int __init scsi_init_devinfo(void) } #ifdef CONFIG_SCSI_PROC_FS - p = proc_create("scsi/device_info", 0, NULL, &scsi_devinfo_proc_fops); + p = proc_create("scsi/device_info", 0, NULL, &scsi_devinfo_proc_ops); if (!p) { error = -ENOMEM; goto out; diff --git a/drivers/scsi/scsi_proc.c b/drivers/scsi/scsi_proc.c index 5b313226f11c..d6982d355739 100644 --- a/drivers/scsi/scsi_proc.c +++ b/drivers/scsi/scsi_proc.c @@ -83,12 +83,12 @@ static int proc_scsi_host_open(struct inode *inode, struct file *file) 4 * PAGE_SIZE); } -static const struct file_operations proc_scsi_fops = { - .open = proc_scsi_host_open, - .release = single_release, - .read = seq_read, - .llseek = seq_lseek, - .write = proc_scsi_host_write +static const struct proc_ops proc_scsi_ops = { + .proc_open = proc_scsi_host_open, + .proc_release = single_release, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_write = proc_scsi_host_write }; /** @@ -146,7 +146,7 @@ void scsi_proc_host_add(struct Scsi_Host *shost) sprintf(name,"%d", shost->host_no); p = proc_create_data(name, S_IRUGO | S_IWUSR, - sht->proc_dir, &proc_scsi_fops, shost); + sht->proc_dir, &proc_scsi_ops, shost); if (!p) printk(KERN_ERR "%s: Failed to register host %d in" "%s\n", __func__, shost->host_no, @@ -436,13 +436,12 @@ static int proc_scsi_open(struct inode *inode, struct file *file) return seq_open(file, &scsi_seq_ops); } -static const struct file_operations proc_scsi_operations = { - .owner = THIS_MODULE, - .open = proc_scsi_open, - .read = seq_read, - .write = proc_scsi_write, - .llseek = seq_lseek, - .release = seq_release, +static const struct proc_ops scsi_scsi_proc_ops = { + .proc_open = proc_scsi_open, + .proc_read = seq_read, + .proc_write = proc_scsi_write, + .proc_lseek = seq_lseek, + .proc_release = seq_release, }; /** @@ -456,7 +455,7 @@ int __init scsi_init_procfs(void) if (!proc_scsi) goto err1; - pde = proc_create("scsi/scsi", 0, NULL, &proc_scsi_operations); + pde = proc_create("scsi/scsi", 0, NULL, &scsi_scsi_proc_ops); if (!pde) goto err2; diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index bafeaf7b9ad8..4e6af592f018 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -2322,25 +2322,23 @@ static int sg_proc_seq_show_int(struct seq_file *s, void *v); static int sg_proc_single_open_adio(struct inode *inode, struct file *file); static ssize_t sg_proc_write_adio(struct file *filp, const char __user *buffer, size_t count, loff_t *off); -static const struct file_operations adio_fops = { - .owner = THIS_MODULE, - .open = sg_proc_single_open_adio, - .read = seq_read, - .llseek = seq_lseek, - .write = sg_proc_write_adio, - .release = single_release, +static const struct proc_ops adio_proc_ops = { + .proc_open = sg_proc_single_open_adio, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_write = sg_proc_write_adio, + .proc_release = single_release, }; static int sg_proc_single_open_dressz(struct inode *inode, struct file *file); static ssize_t sg_proc_write_dressz(struct file *filp, const char __user *buffer, size_t count, loff_t *off); -static const struct file_operations dressz_fops = { - .owner = THIS_MODULE, - .open = sg_proc_single_open_dressz, - .read = seq_read, - .llseek = seq_lseek, - .write = sg_proc_write_dressz, - .release = single_release, +static const struct proc_ops dressz_proc_ops = { + .proc_open = sg_proc_single_open_dressz, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_write = sg_proc_write_dressz, + .proc_release = single_release, }; static int sg_proc_seq_show_version(struct seq_file *s, void *v); @@ -2381,9 +2379,9 @@ sg_proc_init(void) if (!p) return 1; - proc_create("allow_dio", S_IRUGO | S_IWUSR, p, &adio_fops); + proc_create("allow_dio", S_IRUGO | S_IWUSR, p, &adio_proc_ops); proc_create_seq("debug", S_IRUGO, p, &debug_seq_ops); - proc_create("def_reserved_size", S_IRUGO | S_IWUSR, p, &dressz_fops); + proc_create("def_reserved_size", S_IRUGO | S_IWUSR, p, &dressz_proc_ops); proc_create_single("device_hdr", S_IRUGO, p, sg_proc_seq_show_devhdr); proc_create_seq("devices", S_IRUGO, p, &dev_seq_ops); proc_create_seq("device_strs", S_IRUGO, p, &devstrs_seq_ops); diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c index c7266ef295fd..1f59beb7d27e 100644 --- a/drivers/spi/spi-orion.c +++ b/drivers/spi/spi-orion.c @@ -646,8 +646,7 @@ static int orion_spi_probe(struct platform_device *pdev) /* The following clock is only used by some SoCs */ spi->axi_clk = devm_clk_get(&pdev->dev, "axi"); - if (IS_ERR(spi->axi_clk) && - PTR_ERR(spi->axi_clk) == -EPROBE_DEFER) { + if (PTR_ERR(spi->axi_clk) == -EPROBE_DEFER) { status = -EPROBE_DEFER; goto out_rel_clk; } diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig index 642adc4c24d2..c394abffea86 100644 --- a/drivers/staging/media/Kconfig +++ b/drivers/staging/media/Kconfig @@ -38,4 +38,8 @@ source "drivers/staging/media/ipu3/Kconfig" source "drivers/staging/media/soc_camera/Kconfig" +source "drivers/staging/media/phy-rockchip-dphy-rx0/Kconfig" + +source "drivers/staging/media/rkisp1/Kconfig" + endif diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile index 2f1711a8aeed..ea9fce8014bb 100644 --- a/drivers/staging/media/Makefile +++ b/drivers/staging/media/Makefile @@ -8,3 +8,5 @@ obj-$(CONFIG_TEGRA_VDE) += tegra-vde/ obj-$(CONFIG_VIDEO_HANTRO) += hantro/ obj-$(CONFIG_VIDEO_IPU3_IMGU) += ipu3/ obj-$(CONFIG_SOC_CAMERA) += soc_camera/ +obj-$(CONFIG_PHY_ROCKCHIP_DPHY_RX0) += phy-rockchip-dphy-rx0/ +obj-$(CONFIG_VIDEO_ROCKCHIP_ISP1) += rkisp1/ diff --git a/drivers/staging/media/hantro/Makefile b/drivers/staging/media/hantro/Makefile index 5d6b0383d280..496b30c3c396 100644 --- a/drivers/staging/media/hantro/Makefile +++ b/drivers/staging/media/hantro/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_VIDEO_HANTRO) += hantro-vpu.o hantro-vpu-y += \ hantro_drv.o \ hantro_v4l2.o \ + hantro_postproc.o \ hantro_h1_jpeg_enc.o \ hantro_g1_h264_dec.o \ hantro_g1_mpeg2_dec.o \ diff --git a/drivers/staging/media/hantro/hantro.h b/drivers/staging/media/hantro/hantro.h index deb90ae37859..b0faa43b3f79 100644 --- a/drivers/staging/media/hantro/hantro.h +++ b/drivers/staging/media/hantro/hantro.h @@ -60,6 +60,8 @@ struct hantro_irq { * @num_enc_fmts: Number of encoder formats. * @dec_fmts: Decoder formats. * @num_dec_fmts: Number of decoder formats. + * @postproc_fmts: Post-processor formats. + * @num_postproc_fmts: Number of post-processor formats. * @codec: Supported codecs * @codec_ops: Codec ops. * @init: Initialize hardware. @@ -70,6 +72,7 @@ struct hantro_irq { * @num_clocks: number of clocks in the array * @reg_names: array of register range names * @num_regs: number of register range names in the array + * @postproc_regs: &struct hantro_postproc_regs pointer */ struct hantro_variant { unsigned int enc_offset; @@ -78,6 +81,8 @@ struct hantro_variant { unsigned int num_enc_fmts; const struct hantro_fmt *dec_fmts; unsigned int num_dec_fmts; + const struct hantro_fmt *postproc_fmts; + unsigned int num_postproc_fmts; unsigned int codec; const struct hantro_codec_ops *codec_ops; int (*init)(struct hantro_dev *vpu); @@ -88,6 +93,7 @@ struct hantro_variant { int num_clocks; const char * const *reg_names; int num_regs; + const struct hantro_postproc_regs *postproc_regs; }; /** @@ -213,6 +219,7 @@ struct hantro_dev { * context, and it's called right before * calling v4l2_m2m_job_finish. * @codec_ops: Set of operations related to codec mode. + * @postproc: Post-processing context. * @jpeg_enc: JPEG-encoding context. * @mpeg2_dec: MPEG-2-decoding context. * @vp8_dec: VP8-decoding context. @@ -237,6 +244,7 @@ struct hantro_ctx { unsigned int bytesused); const struct hantro_codec_ops *codec_ops; + struct hantro_postproc_ctx postproc; /* Specific for particular codec modes. */ union { @@ -274,6 +282,23 @@ struct hantro_reg { u32 mask; }; +struct hantro_postproc_regs { + struct hantro_reg pipeline_en; + struct hantro_reg max_burst; + struct hantro_reg clk_gate; + struct hantro_reg out_swap32; + struct hantro_reg out_endian; + struct hantro_reg out_luma_base; + struct hantro_reg input_width; + struct hantro_reg input_height; + struct hantro_reg output_width; + struct hantro_reg output_height; + struct hantro_reg input_fmt; + struct hantro_reg output_fmt; + struct hantro_reg orig_width; + struct hantro_reg display_width; +}; + /* Logging helpers */ /** @@ -352,16 +377,30 @@ static inline u32 vdpu_read(struct hantro_dev *vpu, u32 reg) return val; } -static inline void hantro_reg_write(struct hantro_dev *vpu, - const struct hantro_reg *reg, - u32 val) +static inline u32 vdpu_read_mask(struct hantro_dev *vpu, + const struct hantro_reg *reg, + u32 val) { u32 v; v = vdpu_read(vpu, reg->base); v &= ~(reg->mask << reg->shift); v |= ((val & reg->mask) << reg->shift); - vdpu_write_relaxed(vpu, v, reg->base); + return v; +} + +static inline void hantro_reg_write(struct hantro_dev *vpu, + const struct hantro_reg *reg, + u32 val) +{ + vdpu_write_relaxed(vpu, vdpu_read_mask(vpu, reg, val), reg->base); +} + +static inline void hantro_reg_write_s(struct hantro_dev *vpu, + const struct hantro_reg *reg, + u32 val) +{ + vdpu_write(vpu, vdpu_read_mask(vpu, reg, val), reg->base); } bool hantro_is_encoder_ctx(const struct hantro_ctx *ctx); @@ -381,4 +420,23 @@ hantro_get_dst_buf(struct hantro_ctx *ctx) return v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); } +static inline bool +hantro_needs_postproc(struct hantro_ctx *ctx, const struct hantro_fmt *fmt) +{ + return fmt->fourcc != V4L2_PIX_FMT_NV12; +} + +static inline dma_addr_t +hantro_get_dec_buf_addr(struct hantro_ctx *ctx, struct vb2_buffer *vb) +{ + if (hantro_needs_postproc(ctx, ctx->vpu_dst_fmt)) + return ctx->postproc.dec_q[vb->index].dma; + return vb2_dma_contig_plane_dma_addr(vb, 0); +} + +void hantro_postproc_disable(struct hantro_ctx *ctx); +void hantro_postproc_enable(struct hantro_ctx *ctx); +void hantro_postproc_free(struct hantro_ctx *ctx); +int hantro_postproc_alloc(struct hantro_ctx *ctx); + #endif /* HANTRO_H_ */ diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c index 26108c96b674..97c615a2f057 100644 --- a/drivers/staging/media/hantro/hantro_drv.c +++ b/drivers/staging/media/hantro/hantro_drv.c @@ -53,7 +53,7 @@ dma_addr_t hantro_get_ref(struct hantro_ctx *ctx, u64 ts) if (index < 0) return 0; buf = vb2_get_buffer(q, index); - return vb2_dma_contig_plane_dma_addr(buf, 0); + return hantro_get_dec_buf_addr(ctx, buf); } static int @@ -152,16 +152,21 @@ void hantro_watchdog(struct work_struct *work) } } -void hantro_prepare_run(struct hantro_ctx *ctx) +void hantro_start_prepare_run(struct hantro_ctx *ctx) { struct vb2_v4l2_buffer *src_buf; src_buf = hantro_get_src_buf(ctx); v4l2_ctrl_request_setup(src_buf->vb2_buf.req_obj.req, &ctx->ctrl_handler); + + if (hantro_needs_postproc(ctx, ctx->vpu_dst_fmt)) + hantro_postproc_enable(ctx); + else + hantro_postproc_disable(ctx); } -void hantro_finish_run(struct hantro_ctx *ctx) +void hantro_end_prepare_run(struct hantro_ctx *ctx) { struct vb2_v4l2_buffer *src_buf; diff --git a/drivers/staging/media/hantro/hantro_g1_h264_dec.c b/drivers/staging/media/hantro/hantro_g1_h264_dec.c index 3cd40a8f0daa..424c648ce9fc 100644 --- a/drivers/staging/media/hantro/hantro_g1_h264_dec.c +++ b/drivers/staging/media/hantro/hantro_g1_h264_dec.c @@ -244,7 +244,7 @@ static void set_buffers(struct hantro_ctx *ctx) vdpu_write_relaxed(vpu, src_dma, G1_REG_ADDR_STR); /* Destination (decoded frame) buffer. */ - dst_dma = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0); + dst_dma = hantro_get_dec_buf_addr(ctx, &dst_buf->vb2_buf); /* Adjust dma addr to start at second line for bottom field */ if (ctrls->slices[0].flags & V4L2_H264_SLICE_FLAG_BOTTOM_FIELD) offset = ALIGN(ctx->src_fmt.width, MB_DIM); @@ -288,7 +288,7 @@ void hantro_g1_h264_dec_run(struct hantro_ctx *ctx) set_ref(ctx); set_buffers(ctx); - hantro_finish_run(ctx); + hantro_end_prepare_run(ctx); /* Start decoding! */ vdpu_write_relaxed(vpu, diff --git a/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c b/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c index f3bf67d8a289..24041849384a 100644 --- a/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c +++ b/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c @@ -121,7 +121,7 @@ hantro_g1_mpeg2_dec_set_buffers(struct hantro_dev *vpu, struct hantro_ctx *ctx, vdpu_write_relaxed(vpu, addr, G1_REG_RLC_VLC_BASE); /* Destination frame buffer */ - addr = vb2_dma_contig_plane_dma_addr(dst_buf, 0); + addr = hantro_get_dec_buf_addr(ctx, dst_buf); current_addr = addr; if (picture->picture_structure == PICT_BOTTOM_FIELD) @@ -168,7 +168,7 @@ void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) dst_buf = hantro_get_dst_buf(ctx); /* Apply request controls if any */ - hantro_prepare_run(ctx); + hantro_start_prepare_run(ctx); slice_params = hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS); @@ -244,7 +244,7 @@ void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) &dst_buf->vb2_buf, sequence, picture, slice_params); - hantro_finish_run(ctx); + hantro_end_prepare_run(ctx); reg = G1_REG_DEC_E(1); vdpu_write(vpu, reg, G1_SWREG(1)); diff --git a/drivers/staging/media/hantro/hantro_g1_regs.h b/drivers/staging/media/hantro/hantro_g1_regs.h index 5c0ea7994336..c1756e3d5391 100644 --- a/drivers/staging/media/hantro/hantro_g1_regs.h +++ b/drivers/staging/media/hantro/hantro_g1_regs.h @@ -9,6 +9,8 @@ #ifndef HANTRO_G1_REGS_H_ #define HANTRO_G1_REGS_H_ +#define G1_SWREG(nr) ((nr) * 4) + /* Decoder registers. */ #define G1_REG_INTERRUPT 0x004 #define G1_REG_INTERRUPT_DEC_PIC_INF BIT(24) @@ -298,4 +300,55 @@ #define G1_REG_REF_BUF_CTRL2_APF_THRESHOLD(x) (((x) & 0x3fff) << 0) #define G1_REG_SOFT_RESET 0x194 +/* Post-processor registers. */ +#define G1_REG_PP_INTERRUPT G1_SWREG(60) +#define G1_REG_PP_READY_IRQ BIT(12) +#define G1_REG_PP_IRQ BIT(8) +#define G1_REG_PP_IRQ_DIS BIT(4) +#define G1_REG_PP_PIPELINE_EN BIT(1) +#define G1_REG_PP_EXTERNAL_TRIGGER BIT(0) +#define G1_REG_PP_DEV_CONFIG G1_SWREG(61) +#define G1_REG_PP_AXI_RD_ID(v) (((v) << 24) & GENMASK(31, 24)) +#define G1_REG_PP_AXI_WR_ID(v) (((v) << 16) & GENMASK(23, 16)) +#define G1_REG_PP_INSWAP32_E(v) ((v) ? BIT(10) : 0) +#define G1_REG_PP_DATA_DISC_E(v) ((v) ? BIT(9) : 0) +#define G1_REG_PP_CLK_GATE_E(v) ((v) ? BIT(8) : 0) +#define G1_REG_PP_IN_ENDIAN(v) ((v) ? BIT(7) : 0) +#define G1_REG_PP_OUT_ENDIAN(v) ((v) ? BIT(6) : 0) +#define G1_REG_PP_OUTSWAP32_E(v) ((v) ? BIT(5) : 0) +#define G1_REG_PP_MAX_BURST(v) (((v) << 0) & GENMASK(4, 0)) +#define G1_REG_PP_IN_LUMA_BASE G1_SWREG(63) +#define G1_REG_PP_IN_CB_BASE G1_SWREG(64) +#define G1_REG_PP_IN_CR_BASE G1_SWREG(65) +#define G1_REG_PP_OUT_LUMA_BASE G1_SWREG(66) +#define G1_REG_PP_OUT_CHROMA_BASE G1_SWREG(67) +#define G1_REG_PP_CONTRAST_ADJUST G1_SWREG(68) +#define G1_REG_PP_COLOR_CONVERSION G1_SWREG(69) +#define G1_REG_PP_COLOR_CONVERSION0 G1_SWREG(70) +#define G1_REG_PP_COLOR_CONVERSION1 G1_SWREG(71) +#define G1_REG_PP_INPUT_SIZE G1_SWREG(72) +#define G1_REG_PP_INPUT_SIZE_HEIGHT(v) (((v) << 9) & GENMASK(16, 9)) +#define G1_REG_PP_INPUT_SIZE_WIDTH(v) (((v) << 0) & GENMASK(8, 0)) +#define G1_REG_PP_SCALING0 G1_SWREG(79) +#define G1_REG_PP_PADD_R(v) (((v) << 23) & GENMASK(27, 23)) +#define G1_REG_PP_PADD_G(v) (((v) << 18) & GENMASK(22, 18)) +#define G1_REG_PP_RANGEMAP_Y(v) ((v) ? BIT(31) : 0) +#define G1_REG_PP_RANGEMAP_C(v) ((v) ? BIT(30) : 0) +#define G1_REG_PP_YCBCR_RANGE(v) ((v) ? BIT(29) : 0) +#define G1_REG_PP_RGB_16(v) ((v) ? BIT(28) : 0) +#define G1_REG_PP_SCALING1 G1_SWREG(80) +#define G1_REG_PP_PADD_B(v) (((v) << 18) & GENMASK(22, 18)) +#define G1_REG_PP_MASK_R G1_SWREG(82) +#define G1_REG_PP_MASK_G G1_SWREG(83) +#define G1_REG_PP_MASK_B G1_SWREG(84) +#define G1_REG_PP_CONTROL G1_SWREG(85) +#define G1_REG_PP_CONTROL_IN_FMT(v) (((v) << 29) & GENMASK(31, 29)) +#define G1_REG_PP_CONTROL_OUT_FMT(v) (((v) << 26) & GENMASK(28, 26)) +#define G1_REG_PP_CONTROL_OUT_HEIGHT(v) (((v) << 15) & GENMASK(25, 15)) +#define G1_REG_PP_CONTROL_OUT_WIDTH(v) (((v) << 4) & GENMASK(14, 4)) +#define G1_REG_PP_MASK1_ORIG_WIDTH G1_SWREG(88) +#define G1_REG_PP_ORIG_WIDTH(v) (((v) << 23) & GENMASK(31, 23)) +#define G1_REG_PP_DISPLAY_WIDTH G1_SWREG(92) +#define G1_REG_PP_FUSE G1_SWREG(99) + #endif /* HANTRO_G1_REGS_H_ */ diff --git a/drivers/staging/media/hantro/hantro_g1_vp8_dec.c b/drivers/staging/media/hantro/hantro_g1_vp8_dec.c index cad18094fee0..a5cdf150cd16 100644 --- a/drivers/staging/media/hantro/hantro_g1_vp8_dec.c +++ b/drivers/staging/media/hantro/hantro_g1_vp8_dec.c @@ -422,7 +422,7 @@ static void cfg_buffers(struct hantro_ctx *ctx, } vdpu_write_relaxed(vpu, reg, G1_REG_FWD_PIC(0)); - dst_dma = vb2_dma_contig_plane_dma_addr(&vb2_dst->vb2_buf, 0); + dst_dma = hantro_get_dec_buf_addr(ctx, &vb2_dst->vb2_buf); vdpu_write_relaxed(vpu, dst_dma, G1_REG_ADDR_DST); } @@ -435,7 +435,7 @@ void hantro_g1_vp8_dec_run(struct hantro_ctx *ctx) u32 mb_width, mb_height; u32 reg; - hantro_prepare_run(ctx); + hantro_start_prepare_run(ctx); hdr = hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_VP8_FRAME_HEADER); if (WARN_ON(!hdr)) @@ -496,7 +496,7 @@ void hantro_g1_vp8_dec_run(struct hantro_ctx *ctx) cfg_ref(ctx, hdr); cfg_buffers(ctx, hdr); - hantro_finish_run(ctx); + hantro_end_prepare_run(ctx); vdpu_write(vpu, G1_REG_INTERRUPT_DEC_E, G1_REG_INTERRUPT); } diff --git a/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c b/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c index 938b48d4d3d9..0d8afc3e5d71 100644 --- a/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c +++ b/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c @@ -87,7 +87,7 @@ void hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx) src_buf = hantro_get_src_buf(ctx); dst_buf = hantro_get_dst_buf(ctx); - hantro_prepare_run(ctx); + hantro_start_prepare_run(ctx); memset(&jpeg_ctx, 0, sizeof(jpeg_ctx)); jpeg_ctx.buffer = vb2_plane_vaddr(&dst_buf->vb2_buf, 0); @@ -122,7 +122,7 @@ void hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx) | H1_REG_ENC_PIC_INTRA | H1_REG_ENC_CTRL_EN_BIT; - hantro_finish_run(ctx); + hantro_end_prepare_run(ctx); vepu_write(vpu, reg, H1_REG_ENC_CTRL); } diff --git a/drivers/staging/media/hantro/hantro_h264.c b/drivers/staging/media/hantro/hantro_h264.c index 568640eab3a6..f2d3e81fb6ce 100644 --- a/drivers/staging/media/hantro/hantro_h264.c +++ b/drivers/staging/media/hantro/hantro_h264.c @@ -562,7 +562,7 @@ int hantro_h264_dec_prepare_run(struct hantro_ctx *ctx) struct hantro_h264_dec_ctrls *ctrls = &h264_ctx->ctrls; struct hantro_h264_reflist_builder reflist_builder; - hantro_prepare_run(ctx); + hantro_start_prepare_run(ctx); ctrls->scaling = hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX); diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h index fa91dd1848b7..2398d4c1f207 100644 --- a/drivers/staging/media/hantro/hantro_hw.h +++ b/drivers/staging/media/hantro/hantro_hw.h @@ -28,11 +28,13 @@ struct hantro_variant; * @cpu: CPU pointer to the buffer. * @dma: DMA address of the buffer. * @size: Size of the buffer. + * @attrs: Attributes of the DMA mapping. */ struct hantro_aux_buf { void *cpu; dma_addr_t dma; size_t size; + unsigned long attrs; }; /** @@ -107,6 +109,15 @@ struct hantro_vp8_dec_hw_ctx { }; /** + * struct hantro_postproc_ctx + * + * @dec_q: References buffers, in decoder format. + */ +struct hantro_postproc_ctx { + struct hantro_aux_buf dec_q[VB2_MAX_FRAME]; +}; + +/** * struct hantro_codec_ops - codec mode specific operations * * @init: If needed, can be used for initialization. @@ -141,14 +152,16 @@ extern const struct hantro_variant rk3399_vpu_variant; extern const struct hantro_variant rk3328_vpu_variant; extern const struct hantro_variant rk3288_vpu_variant; +extern const struct hantro_postproc_regs hantro_g1_postproc_regs; + extern const u32 hantro_vp8_dec_mc_filter[8][6]; void hantro_watchdog(struct work_struct *work); void hantro_run(struct hantro_ctx *ctx); void hantro_irq_done(struct hantro_dev *vpu, unsigned int bytesused, enum vb2_buffer_state result); -void hantro_prepare_run(struct hantro_ctx *ctx); -void hantro_finish_run(struct hantro_ctx *ctx); +void hantro_start_prepare_run(struct hantro_ctx *ctx); +void hantro_end_prepare_run(struct hantro_ctx *ctx); void hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx); void rk3399_vpu_jpeg_enc_run(struct hantro_ctx *ctx); diff --git a/drivers/staging/media/hantro/hantro_postproc.c b/drivers/staging/media/hantro/hantro_postproc.c new file mode 100644 index 000000000000..28a85d301d7f --- /dev/null +++ b/drivers/staging/media/hantro/hantro_postproc.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hantro G1 post-processor support + * + * Copyright (C) 2019 Collabora, Ltd. + */ + +#include <linux/dma-mapping.h> +#include <linux/types.h> + +#include "hantro.h" +#include "hantro_hw.h" +#include "hantro_g1_regs.h" + +#define HANTRO_PP_REG_WRITE(vpu, reg_name, val) \ +{ \ + hantro_reg_write((vpu), \ + &((vpu)->variant->postproc_regs->reg_name), \ + (val)); \ +} + +#define HANTRO_PP_REG_WRITE_S(vpu, reg_name, val) \ +{ \ + hantro_reg_write_s((vpu), \ + &((vpu)->variant->postproc_regs->reg_name), \ + (val)); \ +} + +#define VPU_PP_IN_YUYV 0x0 +#define VPU_PP_IN_NV12 0x1 +#define VPU_PP_IN_YUV420 0x2 +#define VPU_PP_IN_YUV240_TILED 0x5 +#define VPU_PP_OUT_RGB 0x0 +#define VPU_PP_OUT_YUYV 0x3 + +const struct hantro_postproc_regs hantro_g1_postproc_regs = { + .pipeline_en = {G1_REG_PP_INTERRUPT, 1, 0x1}, + .max_burst = {G1_REG_PP_DEV_CONFIG, 0, 0x1f}, + .clk_gate = {G1_REG_PP_DEV_CONFIG, 1, 0x1}, + .out_swap32 = {G1_REG_PP_DEV_CONFIG, 5, 0x1}, + .out_endian = {G1_REG_PP_DEV_CONFIG, 6, 0x1}, + .out_luma_base = {G1_REG_PP_OUT_LUMA_BASE, 0, 0xffffffff}, + .input_width = {G1_REG_PP_INPUT_SIZE, 0, 0x1ff}, + .input_height = {G1_REG_PP_INPUT_SIZE, 9, 0x1ff}, + .output_width = {G1_REG_PP_CONTROL, 4, 0x7ff}, + .output_height = {G1_REG_PP_CONTROL, 15, 0x7ff}, + .input_fmt = {G1_REG_PP_CONTROL, 29, 0x7}, + .output_fmt = {G1_REG_PP_CONTROL, 26, 0x7}, + .orig_width = {G1_REG_PP_MASK1_ORIG_WIDTH, 23, 0x1ff}, + .display_width = {G1_REG_PP_DISPLAY_WIDTH, 0, 0xfff}, +}; + +void hantro_postproc_enable(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + struct vb2_v4l2_buffer *dst_buf; + u32 src_pp_fmt, dst_pp_fmt; + dma_addr_t dst_dma; + + if (!vpu->variant->postproc_regs) + return; + + /* Turn on pipeline mode. Must be done first. */ + HANTRO_PP_REG_WRITE_S(vpu, pipeline_en, 0x1); + + src_pp_fmt = VPU_PP_IN_NV12; + + switch (ctx->vpu_dst_fmt->fourcc) { + case V4L2_PIX_FMT_YUYV: + dst_pp_fmt = VPU_PP_OUT_YUYV; + break; + default: + WARN(1, "output format %d not supported by the post-processor, this wasn't expected.", + ctx->vpu_dst_fmt->fourcc); + dst_pp_fmt = 0; + break; + } + + dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + dst_dma = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0); + + HANTRO_PP_REG_WRITE(vpu, clk_gate, 0x1); + HANTRO_PP_REG_WRITE(vpu, out_endian, 0x1); + HANTRO_PP_REG_WRITE(vpu, out_swap32, 0x1); + HANTRO_PP_REG_WRITE(vpu, max_burst, 16); + HANTRO_PP_REG_WRITE(vpu, out_luma_base, dst_dma); + HANTRO_PP_REG_WRITE(vpu, input_width, MB_WIDTH(ctx->dst_fmt.width)); + HANTRO_PP_REG_WRITE(vpu, input_height, MB_HEIGHT(ctx->dst_fmt.height)); + HANTRO_PP_REG_WRITE(vpu, input_fmt, src_pp_fmt); + HANTRO_PP_REG_WRITE(vpu, output_fmt, dst_pp_fmt); + HANTRO_PP_REG_WRITE(vpu, output_width, ctx->dst_fmt.width); + HANTRO_PP_REG_WRITE(vpu, output_height, ctx->dst_fmt.height); + HANTRO_PP_REG_WRITE(vpu, orig_width, MB_WIDTH(ctx->dst_fmt.width)); + HANTRO_PP_REG_WRITE(vpu, display_width, ctx->dst_fmt.width); +} + +void hantro_postproc_free(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + unsigned int i; + + for (i = 0; i < VB2_MAX_FRAME; ++i) { + struct hantro_aux_buf *priv = &ctx->postproc.dec_q[i]; + + if (priv->cpu) { + dma_free_attrs(vpu->dev, priv->size, priv->cpu, + priv->dma, priv->attrs); + priv->cpu = NULL; + } + } +} + +int hantro_postproc_alloc(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + struct v4l2_m2m_ctx *m2m_ctx = ctx->fh.m2m_ctx; + struct vb2_queue *cap_queue = &m2m_ctx->cap_q_ctx.q; + unsigned int num_buffers = cap_queue->num_buffers; + unsigned int i, buf_size; + + buf_size = ctx->dst_fmt.plane_fmt[0].sizeimage; + + for (i = 0; i < num_buffers; ++i) { + struct hantro_aux_buf *priv = &ctx->postproc.dec_q[i]; + + /* + * The buffers on this queue are meant as intermediate + * buffers for the decoder, so no mapping is needed. + */ + priv->attrs = DMA_ATTR_NO_KERNEL_MAPPING; + priv->cpu = dma_alloc_attrs(vpu->dev, buf_size, &priv->dma, + GFP_KERNEL, priv->attrs); + if (!priv->cpu) + return -ENOMEM; + priv->size = buf_size; + } + return 0; +} + +void hantro_postproc_disable(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + + if (!vpu->variant->postproc_regs) + return; + + HANTRO_PP_REG_WRITE_S(vpu, pipeline_en, 0x0); +} diff --git a/drivers/staging/media/hantro/hantro_v4l2.c b/drivers/staging/media/hantro/hantro_v4l2.c index 1dae76f20034..0198bcda26b7 100644 --- a/drivers/staging/media/hantro/hantro_v4l2.c +++ b/drivers/staging/media/hantro/hantro_v4l2.c @@ -47,11 +47,30 @@ hantro_get_formats(const struct hantro_ctx *ctx, unsigned int *num_fmts) } static const struct hantro_fmt * -hantro_find_format(const struct hantro_fmt *formats, unsigned int num_fmts, - u32 fourcc) +hantro_get_postproc_formats(const struct hantro_ctx *ctx, + unsigned int *num_fmts) { - unsigned int i; + if (hantro_is_encoder_ctx(ctx)) { + *num_fmts = 0; + return NULL; + } + + *num_fmts = ctx->dev->variant->num_postproc_fmts; + return ctx->dev->variant->postproc_fmts; +} + +static const struct hantro_fmt * +hantro_find_format(const struct hantro_ctx *ctx, u32 fourcc) +{ + const struct hantro_fmt *formats; + unsigned int i, num_fmts; + + formats = hantro_get_formats(ctx, &num_fmts); + for (i = 0; i < num_fmts; i++) + if (formats[i].fourcc == fourcc) + return &formats[i]; + formats = hantro_get_postproc_formats(ctx, &num_fmts); for (i = 0; i < num_fmts; i++) if (formats[i].fourcc == fourcc) return &formats[i]; @@ -59,11 +78,12 @@ hantro_find_format(const struct hantro_fmt *formats, unsigned int num_fmts, } static const struct hantro_fmt * -hantro_get_default_fmt(const struct hantro_fmt *formats, unsigned int num_fmts, - bool bitstream) +hantro_get_default_fmt(const struct hantro_ctx *ctx, bool bitstream) { - unsigned int i; + const struct hantro_fmt *formats; + unsigned int i, num_fmts; + formats = hantro_get_formats(ctx, &num_fmts); for (i = 0; i < num_fmts; i++) { if (bitstream == (formats[i].codec_mode != HANTRO_MODE_NONE)) @@ -89,8 +109,7 @@ static int vidioc_enum_framesizes(struct file *file, void *priv, struct v4l2_frmsizeenum *fsize) { struct hantro_ctx *ctx = fh_to_ctx(priv); - const struct hantro_fmt *formats, *fmt; - unsigned int num_fmts; + const struct hantro_fmt *fmt; if (fsize->index != 0) { vpu_debug(0, "invalid frame size index (expected 0, got %d)\n", @@ -98,8 +117,7 @@ static int vidioc_enum_framesizes(struct file *file, void *priv, return -EINVAL; } - formats = hantro_get_formats(ctx, &num_fmts); - fmt = hantro_find_format(formats, num_fmts, fsize->pixel_format); + fmt = hantro_find_format(ctx, fsize->pixel_format); if (!fmt) { vpu_debug(0, "unsupported bitstream format (%08x)\n", fsize->pixel_format); @@ -150,6 +168,24 @@ static int vidioc_enum_fmt(struct file *file, void *priv, } ++j; } + + /* + * Enumerate post-processed formats. As per the specification, + * we enumerated these formats after natively decoded formats + * as a hint for applications on what's the preferred fomat. + */ + if (!capture) + return -EINVAL; + formats = hantro_get_postproc_formats(ctx, &num_fmts); + for (i = 0; i < num_fmts; i++) { + if (j == f->index) { + fmt = &formats[i]; + f->pixelformat = fmt->fourcc; + return 0; + } + ++j; + } + return -EINVAL; } @@ -196,8 +232,7 @@ static int vidioc_try_fmt(struct file *file, void *priv, struct v4l2_format *f, { struct hantro_ctx *ctx = fh_to_ctx(priv); struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp; - const struct hantro_fmt *formats, *fmt, *vpu_fmt; - unsigned int num_fmts; + const struct hantro_fmt *fmt, *vpu_fmt; bool coded; coded = capture == hantro_is_encoder_ctx(ctx); @@ -208,10 +243,9 @@ static int vidioc_try_fmt(struct file *file, void *priv, struct v4l2_format *f, (pix_mp->pixelformat >> 16) & 0x7f, (pix_mp->pixelformat >> 24) & 0x7f); - formats = hantro_get_formats(ctx, &num_fmts); - fmt = hantro_find_format(formats, num_fmts, pix_mp->pixelformat); + fmt = hantro_find_format(ctx, pix_mp->pixelformat); if (!fmt) { - fmt = hantro_get_default_fmt(formats, num_fmts, coded); + fmt = hantro_get_default_fmt(ctx, coded); f->fmt.pix_mp.pixelformat = fmt->fourcc; } @@ -246,7 +280,7 @@ static int vidioc_try_fmt(struct file *file, void *priv, struct v4l2_format *f, * * The H264 decoder needs extra space on the output buffers * to store motion vectors. This is needed for reference - * frames. + * frames and only if the format is non-post-processed NV12. * * Memory layout is as follow: * @@ -260,7 +294,8 @@ static int vidioc_try_fmt(struct file *file, void *priv, struct v4l2_format *f, * | MC sync 32 bytes | * +---------------------------+ */ - if (ctx->vpu_src_fmt->fourcc == V4L2_PIX_FMT_H264_SLICE) + if (ctx->vpu_src_fmt->fourcc == V4L2_PIX_FMT_H264_SLICE && + !hantro_needs_postproc(ctx, ctx->vpu_dst_fmt)) pix_mp->plane_fmt[0].sizeimage += 64 * MB_WIDTH(pix_mp->width) * MB_WIDTH(pix_mp->height) + 32; @@ -306,12 +341,10 @@ hantro_reset_fmt(struct v4l2_pix_format_mplane *fmt, static void hantro_reset_encoded_fmt(struct hantro_ctx *ctx) { - const struct hantro_fmt *vpu_fmt, *formats; + const struct hantro_fmt *vpu_fmt; struct v4l2_pix_format_mplane *fmt; - unsigned int num_fmts; - formats = hantro_get_formats(ctx, &num_fmts); - vpu_fmt = hantro_get_default_fmt(formats, num_fmts, true); + vpu_fmt = hantro_get_default_fmt(ctx, true); if (hantro_is_encoder_ctx(ctx)) { ctx->vpu_dst_fmt = vpu_fmt; @@ -332,12 +365,10 @@ hantro_reset_encoded_fmt(struct hantro_ctx *ctx) static void hantro_reset_raw_fmt(struct hantro_ctx *ctx) { - const struct hantro_fmt *raw_vpu_fmt, *formats; + const struct hantro_fmt *raw_vpu_fmt; struct v4l2_pix_format_mplane *raw_fmt, *encoded_fmt; - unsigned int num_fmts; - formats = hantro_get_formats(ctx, &num_fmts); - raw_vpu_fmt = hantro_get_default_fmt(formats, num_fmts, false); + raw_vpu_fmt = hantro_get_default_fmt(ctx, false); if (hantro_is_encoder_ctx(ctx)) { ctx->vpu_src_fmt = raw_vpu_fmt; @@ -384,8 +415,6 @@ vidioc_s_fmt_out_mplane(struct file *file, void *priv, struct v4l2_format *f) struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp; struct hantro_ctx *ctx = fh_to_ctx(priv); struct vb2_queue *vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); - const struct hantro_fmt *formats; - unsigned int num_fmts; int ret; ret = vidioc_try_fmt_out_mplane(file, priv, f); @@ -421,9 +450,7 @@ vidioc_s_fmt_out_mplane(struct file *file, void *priv, struct v4l2_format *f) return -EBUSY; } - formats = hantro_get_formats(ctx, &num_fmts); - ctx->vpu_src_fmt = hantro_find_format(formats, num_fmts, - pix_mp->pixelformat); + ctx->vpu_src_fmt = hantro_find_format(ctx, pix_mp->pixelformat); ctx->src_fmt = *pix_mp; /* @@ -457,9 +484,7 @@ static int vidioc_s_fmt_cap_mplane(struct file *file, void *priv, { struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp; struct hantro_ctx *ctx = fh_to_ctx(priv); - const struct hantro_fmt *formats; struct vb2_queue *vq; - unsigned int num_fmts; int ret; /* Change not allowed if queue is busy. */ @@ -488,9 +513,7 @@ static int vidioc_s_fmt_cap_mplane(struct file *file, void *priv, if (ret) return ret; - formats = hantro_get_formats(ctx, &num_fmts); - ctx->vpu_dst_fmt = hantro_find_format(formats, num_fmts, - pix_mp->pixelformat); + ctx->vpu_dst_fmt = hantro_find_format(ctx, pix_mp->pixelformat); ctx->dst_fmt = *pix_mp; /* @@ -650,10 +673,23 @@ static int hantro_start_streaming(struct vb2_queue *q, unsigned int count) vpu_debug(4, "Codec mode = %d\n", codec_mode); ctx->codec_ops = &ctx->dev->variant->codec_ops[codec_mode]; - if (ctx->codec_ops->init) + if (ctx->codec_ops->init) { ret = ctx->codec_ops->init(ctx); + if (ret) + return ret; + } + + if (hantro_needs_postproc(ctx, ctx->vpu_dst_fmt)) { + ret = hantro_postproc_alloc(ctx); + if (ret) + goto err_codec_exit; + } } + return ret; +err_codec_exit: + if (ctx->codec_ops->exit) + ctx->codec_ops->exit(ctx); return ret; } @@ -680,6 +716,7 @@ static void hantro_stop_streaming(struct vb2_queue *q) struct hantro_ctx *ctx = vb2_get_drv_priv(q); if (hantro_vq_is_coded(q)) { + hantro_postproc_free(ctx); if (ctx->codec_ops && ctx->codec_ops->exit) ctx->codec_ops->exit(ctx); } diff --git a/drivers/staging/media/hantro/rk3288_vpu_hw.c b/drivers/staging/media/hantro/rk3288_vpu_hw.c index f8db6fcaad73..2f914b37b9e5 100644 --- a/drivers/staging/media/hantro/rk3288_vpu_hw.c +++ b/drivers/staging/media/hantro/rk3288_vpu_hw.c @@ -56,6 +56,13 @@ static const struct hantro_fmt rk3288_vpu_enc_fmts[] = { }, }; +static const struct hantro_fmt rk3288_vpu_postproc_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_YUYV, + .codec_mode = HANTRO_MODE_NONE, + }, +}; + static const struct hantro_fmt rk3288_vpu_dec_fmts[] = { { .fourcc = V4L2_PIX_FMT_NV12, @@ -215,6 +222,9 @@ const struct hantro_variant rk3288_vpu_variant = { .dec_offset = 0x400, .dec_fmts = rk3288_vpu_dec_fmts, .num_dec_fmts = ARRAY_SIZE(rk3288_vpu_dec_fmts), + .postproc_fmts = rk3288_vpu_postproc_fmts, + .num_postproc_fmts = ARRAY_SIZE(rk3288_vpu_postproc_fmts), + .postproc_regs = &hantro_g1_postproc_regs, .codec = HANTRO_JPEG_ENCODER | HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER | HANTRO_H264_DECODER, .codec_ops = rk3288_vpu_codec_ops, diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c b/drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c index 067892345b5d..4c2d43fb6fd1 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c +++ b/drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c @@ -118,7 +118,7 @@ void rk3399_vpu_jpeg_enc_run(struct hantro_ctx *ctx) src_buf = hantro_get_src_buf(ctx); dst_buf = hantro_get_dst_buf(ctx); - hantro_prepare_run(ctx); + hantro_start_prepare_run(ctx); memset(&jpeg_ctx, 0, sizeof(jpeg_ctx)); jpeg_ctx.buffer = vb2_plane_vaddr(&dst_buf->vb2_buf, 0); @@ -156,6 +156,6 @@ void rk3399_vpu_jpeg_enc_run(struct hantro_ctx *ctx) | VEPU_REG_ENCODE_ENABLE; /* Kick the watchdog and start encoding */ - hantro_finish_run(ctx); + hantro_end_prepare_run(ctx); vepu_write(vpu, reg, VEPU_REG_ENCODE_START); } diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c b/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c index b40d2cdf832f..7e9aad671489 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c +++ b/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c @@ -169,7 +169,7 @@ void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) src_buf = hantro_get_src_buf(ctx); dst_buf = hantro_get_dst_buf(ctx); - hantro_prepare_run(ctx); + hantro_start_prepare_run(ctx); slice_params = hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS); @@ -250,7 +250,7 @@ void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) sequence, picture, slice_params); /* Kick the watchdog and start decoding */ - hantro_finish_run(ctx); + hantro_end_prepare_run(ctx); reg = vdpu_read(vpu, VDPU_SWREG(57)) | VDPU_REG_DEC_E(1); vdpu_write(vpu, reg, VDPU_SWREG(57)); diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c b/drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c index 76d7ed3fd69a..a4a792f00b11 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c +++ b/drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c @@ -513,7 +513,7 @@ void rk3399_vpu_vp8_dec_run(struct hantro_ctx *ctx) u32 mb_width, mb_height; u32 reg; - hantro_prepare_run(ctx); + hantro_start_prepare_run(ctx); hdr = hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_VP8_FRAME_HEADER); if (WARN_ON(!hdr)) @@ -587,7 +587,7 @@ void rk3399_vpu_vp8_dec_run(struct hantro_ctx *ctx) cfg_ref(ctx, hdr); cfg_buffers(ctx, hdr); - hantro_finish_run(ctx); + hantro_end_prepare_run(ctx); hantro_reg_write(vpu, &vp8_dec_start_dec, 1); } diff --git a/drivers/staging/media/imx/imx7-mipi-csis.c b/drivers/staging/media/imx/imx7-mipi-csis.c index 99166afca071..383abecb3bec 100644 --- a/drivers/staging/media/imx/imx7-mipi-csis.c +++ b/drivers/staging/media/imx/imx7-mipi-csis.c @@ -251,8 +251,6 @@ struct csi_state { struct mipi_csis_event events[MIPI_CSIS_NUM_EVENTS]; - struct v4l2_async_notifier subdev_notifier; - struct csis_hw_reset hw_reset; struct regulator *mipi_phy_regulator; bool sink_linked; @@ -1104,7 +1102,6 @@ static int mipi_csis_remove(struct platform_device *pdev) mipi_csis_debugfs_exit(state); v4l2_async_unregister_subdev(&state->mipi_sd); - v4l2_async_notifier_unregister(&state->subdev_notifier); pm_runtime_disable(&pdev->dev); mipi_csis_pm_suspend(&pdev->dev, true); diff --git a/drivers/staging/media/ipu3/TODO b/drivers/staging/media/ipu3/TODO index b44bb4a72ca7..dc356d6f03c4 100644 --- a/drivers/staging/media/ipu3/TODO +++ b/drivers/staging/media/ipu3/TODO @@ -5,15 +5,9 @@ staging directory. as well as formats and the binary used to a request. Remove the opportunistic buffer management. (Sakari) -- Using ENABLED and IMMUTABLE link flags for the links where those are - relevant. (Sakari) - - IPU3 driver documentation (Laurent) Comments on configuring v4l2 subdevs for CIO2 and ImgU. -- uAPI documentation: - Move acronyms to doc-rst file. (Mauro) - - Switch to yavta from v4l2n in driver docs. - Elaborate the functionality of different selection rectangles in driver diff --git a/drivers/staging/media/ipu3/ipu3-css.c b/drivers/staging/media/ipu3/ipu3-css.c index fd1ed84c400c..f36de501edc6 100644 --- a/drivers/staging/media/ipu3/ipu3-css.c +++ b/drivers/staging/media/ipu3/ipu3-css.c @@ -1450,7 +1450,7 @@ bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe) bool imgu_css_queue_empty(struct imgu_css *css) { unsigned int pipe; - bool ret = 0; + bool ret = false; for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) ret &= imgu_css_pipe_queue_empty(css, pipe); diff --git a/drivers/staging/media/ipu3/ipu3-v4l2.c b/drivers/staging/media/ipu3/ipu3-v4l2.c index 3c7ad1eed434..569e27b824c8 100644 --- a/drivers/staging/media/ipu3/ipu3-v4l2.c +++ b/drivers/staging/media/ipu3/ipu3-v4l2.c @@ -1260,6 +1260,11 @@ static int imgu_v4l2_node_setup(struct imgu_device *imgu, unsigned int pipe, r = media_create_pad_link(&vdev->entity, 0, &sd->entity, node_num, flags); } else { + if (node->id == IMGU_NODE_OUT) { + flags |= MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE; + node->enabled = true; + } + r = media_create_pad_link(&sd->entity, node_num, &vdev->entity, 0, flags); } diff --git a/drivers/staging/media/meson/vdec/vdec.c b/drivers/staging/media/meson/vdec/vdec.c index 0a1a04fd5d13..5c5dabed2f09 100644 --- a/drivers/staging/media/meson/vdec/vdec.c +++ b/drivers/staging/media/meson/vdec/vdec.c @@ -133,6 +133,8 @@ vdec_queue_recycle(struct amvdec_session *sess, struct vb2_buffer *vb) struct amvdec_buffer *new_buf; new_buf = kmalloc(sizeof(*new_buf), GFP_KERNEL); + if (!new_buf) + return; new_buf->vb = vb; mutex_lock(&sess->bufs_recycle_lock); @@ -956,6 +958,10 @@ static const struct of_device_id vdec_dt_match[] = { .data = &vdec_platform_gxm }, { .compatible = "amlogic,gxl-vdec", .data = &vdec_platform_gxl }, + { .compatible = "amlogic,g12a-vdec", + .data = &vdec_platform_g12a }, + { .compatible = "amlogic,sm1-vdec", + .data = &vdec_platform_sm1 }, {} }; MODULE_DEVICE_TABLE(of, vdec_dt_match); @@ -1003,6 +1009,16 @@ static int vdec_probe(struct platform_device *pdev) if (IS_ERR(core->canvas)) return PTR_ERR(core->canvas); + of_id = of_match_node(vdec_dt_match, dev->of_node); + core->platform = of_id->data; + + if (core->platform->revision == VDEC_REVISION_G12A || + core->platform->revision == VDEC_REVISION_SM1) { + core->vdec_hevcf_clk = devm_clk_get(dev, "vdec_hevcf"); + if (IS_ERR(core->vdec_hevcf_clk)) + return -EPROBE_DEFER; + } + core->dos_parser_clk = devm_clk_get(dev, "dos_parser"); if (IS_ERR(core->dos_parser_clk)) return -EPROBE_DEFER; @@ -1045,8 +1061,6 @@ static int vdec_probe(struct platform_device *pdev) goto err_vdev_release; } - of_id = of_match_node(vdec_dt_match, dev->of_node); - core->platform = of_id->data; core->vdev_dec = vdev; core->dev_dec = dev; mutex_init(&core->lock); diff --git a/drivers/staging/media/meson/vdec/vdec.h b/drivers/staging/media/meson/vdec/vdec.h index d811e7976519..0faa1ec4858e 100644 --- a/drivers/staging/media/meson/vdec/vdec.h +++ b/drivers/staging/media/meson/vdec/vdec.h @@ -74,6 +74,7 @@ struct amvdec_core { struct clk *dos_clk; struct clk *vdec_1_clk; struct clk *vdec_hevc_clk; + struct clk *vdec_hevcf_clk; struct reset_control *esparser_reset; diff --git a/drivers/staging/media/meson/vdec/vdec_1.c b/drivers/staging/media/meson/vdec/vdec_1.c index 3a15c6fc0567..3fe2de0c9331 100644 --- a/drivers/staging/media/meson/vdec/vdec_1.c +++ b/drivers/staging/media/meson/vdec/vdec_1.c @@ -18,6 +18,7 @@ #define AO_RTI_GEN_PWR_SLEEP0 0xe8 #define AO_RTI_GEN_PWR_ISO0 0xec #define GEN_PWR_VDEC_1 (BIT(3) | BIT(2)) + #define GEN_PWR_VDEC_1_SM1 (BIT(1)) #define MC_SIZE (4096 * 4) @@ -142,12 +143,20 @@ static int vdec_1_stop(struct amvdec_session *sess) amvdec_read_dos(core, DOS_SW_RESET0); /* enable vdec1 isolation */ - regmap_write(core->regmap_ao, AO_RTI_GEN_PWR_ISO0, 0xc0); + if (core->platform->revision == VDEC_REVISION_SM1) + regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_ISO0, + GEN_PWR_VDEC_1_SM1, GEN_PWR_VDEC_1_SM1); + else + regmap_write(core->regmap_ao, AO_RTI_GEN_PWR_ISO0, 0xc0); /* power off vdec1 memories */ amvdec_write_dos(core, DOS_MEM_PD_VDEC, 0xffffffff); /* power off vdec1 */ - regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_SLEEP0, - GEN_PWR_VDEC_1, GEN_PWR_VDEC_1); + if (core->platform->revision == VDEC_REVISION_SM1) + regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_SLEEP0, + GEN_PWR_VDEC_1_SM1, GEN_PWR_VDEC_1_SM1); + else + regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_SLEEP0, + GEN_PWR_VDEC_1, GEN_PWR_VDEC_1); clk_disable_unprepare(core->vdec_1_clk); @@ -170,8 +179,12 @@ static int vdec_1_start(struct amvdec_session *sess) return ret; /* Enable power for VDEC_1 */ - regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_SLEEP0, - GEN_PWR_VDEC_1, 0); + if (core->platform->revision == VDEC_REVISION_SM1) + regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_SLEEP0, + GEN_PWR_VDEC_1_SM1, 0); + else + regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_SLEEP0, + GEN_PWR_VDEC_1, 0); usleep_range(10, 20); /* Reset VDEC1 */ @@ -183,7 +196,11 @@ static int vdec_1_start(struct amvdec_session *sess) /* enable VDEC Memories */ amvdec_write_dos(core, DOS_MEM_PD_VDEC, 0); /* Remove VDEC1 Isolation */ - regmap_write(core->regmap_ao, AO_RTI_GEN_PWR_ISO0, 0); + if (core->platform->revision == VDEC_REVISION_SM1) + regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_ISO0, + GEN_PWR_VDEC_1_SM1, 0); + else + regmap_write(core->regmap_ao, AO_RTI_GEN_PWR_ISO0, 0); /* Reset DOS top registers */ amvdec_write_dos(core, DOS_VDEC_MCRCC_STALL_CTRL, 0); diff --git a/drivers/staging/media/meson/vdec/vdec_platform.c b/drivers/staging/media/meson/vdec/vdec_platform.c index 824dbc7f46f5..ea39f8209ec7 100644 --- a/drivers/staging/media/meson/vdec/vdec_platform.c +++ b/drivers/staging/media/meson/vdec/vdec_platform.c @@ -82,6 +82,54 @@ static const struct amvdec_format vdec_formats_gxm[] = { }, }; +static const struct amvdec_format vdec_formats_g12a[] = { + { + .pixfmt = V4L2_PIX_FMT_MPEG1, + .min_buffers = 8, + .max_buffers = 8, + .max_width = 1920, + .max_height = 1080, + .vdec_ops = &vdec_1_ops, + .codec_ops = &codec_mpeg12_ops, + .firmware_path = "meson/vdec/gxl_mpeg12.bin", + .pixfmts_cap = { V4L2_PIX_FMT_NV12M, V4L2_PIX_FMT_YUV420M, 0 }, + }, { + .pixfmt = V4L2_PIX_FMT_MPEG2, + .min_buffers = 8, + .max_buffers = 8, + .max_width = 1920, + .max_height = 1080, + .vdec_ops = &vdec_1_ops, + .codec_ops = &codec_mpeg12_ops, + .firmware_path = "meson/vdec/gxl_mpeg12.bin", + .pixfmts_cap = { V4L2_PIX_FMT_NV12M, V4L2_PIX_FMT_YUV420M, 0 }, + }, +}; + +static const struct amvdec_format vdec_formats_sm1[] = { + { + .pixfmt = V4L2_PIX_FMT_MPEG1, + .min_buffers = 8, + .max_buffers = 8, + .max_width = 1920, + .max_height = 1080, + .vdec_ops = &vdec_1_ops, + .codec_ops = &codec_mpeg12_ops, + .firmware_path = "meson/vdec/gxl_mpeg12.bin", + .pixfmts_cap = { V4L2_PIX_FMT_NV12M, V4L2_PIX_FMT_YUV420M, 0 }, + }, { + .pixfmt = V4L2_PIX_FMT_MPEG2, + .min_buffers = 8, + .max_buffers = 8, + .max_width = 1920, + .max_height = 1080, + .vdec_ops = &vdec_1_ops, + .codec_ops = &codec_mpeg12_ops, + .firmware_path = "meson/vdec/gxl_mpeg12.bin", + .pixfmts_cap = { V4L2_PIX_FMT_NV12M, V4L2_PIX_FMT_YUV420M, 0 }, + }, +}; + const struct vdec_platform vdec_platform_gxbb = { .formats = vdec_formats_gxbb, .num_formats = ARRAY_SIZE(vdec_formats_gxbb), @@ -99,3 +147,15 @@ const struct vdec_platform vdec_platform_gxm = { .num_formats = ARRAY_SIZE(vdec_formats_gxm), .revision = VDEC_REVISION_GXM, }; + +const struct vdec_platform vdec_platform_g12a = { + .formats = vdec_formats_g12a, + .num_formats = ARRAY_SIZE(vdec_formats_g12a), + .revision = VDEC_REVISION_G12A, +}; + +const struct vdec_platform vdec_platform_sm1 = { + .formats = vdec_formats_sm1, + .num_formats = ARRAY_SIZE(vdec_formats_sm1), + .revision = VDEC_REVISION_SM1, +}; diff --git a/drivers/staging/media/meson/vdec/vdec_platform.h b/drivers/staging/media/meson/vdec/vdec_platform.h index f6025326db1d..731877a771f4 100644 --- a/drivers/staging/media/meson/vdec/vdec_platform.h +++ b/drivers/staging/media/meson/vdec/vdec_platform.h @@ -15,6 +15,8 @@ enum vdec_revision { VDEC_REVISION_GXBB, VDEC_REVISION_GXL, VDEC_REVISION_GXM, + VDEC_REVISION_G12A, + VDEC_REVISION_SM1, }; struct vdec_platform { @@ -26,5 +28,7 @@ struct vdec_platform { extern const struct vdec_platform vdec_platform_gxbb; extern const struct vdec_platform vdec_platform_gxm; extern const struct vdec_platform vdec_platform_gxl; +extern const struct vdec_platform vdec_platform_g12a; +extern const struct vdec_platform vdec_platform_sm1; #endif diff --git a/drivers/staging/media/phy-rockchip-dphy-rx0/Documentation/devicetree/bindings/phy/rockchip-mipi-dphy-rx0.yaml b/drivers/staging/media/phy-rockchip-dphy-rx0/Documentation/devicetree/bindings/phy/rockchip-mipi-dphy-rx0.yaml new file mode 100644 index 000000000000..5dacece35702 --- /dev/null +++ b/drivers/staging/media/phy-rockchip-dphy-rx0/Documentation/devicetree/bindings/phy/rockchip-mipi-dphy-rx0.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/rockchip-mipi-dphy-rx0.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip SoC MIPI RX0 D-PHY Device Tree Bindings + +maintainers: + - Helen Koike <helen.koike@collabora.com> + - Ezequiel Garcia <ezequiel@collabora.com> + +description: | + The Rockchip SoC has a MIPI D-PHY bus with an RX0 entry which connects to + the ISP1 (Image Signal Processing unit v1.0) for CSI cameras. + +properties: + compatible: + const: rockchip,rk3399-mipi-dphy-rx0 + + reg: + maxItems: 1 + + clocks: + items: + - description: MIPI D-PHY ref clock + - description: MIPI D-PHY RX0 cfg clock + - description: Video in/out general register file clock + + clock-names: + items: + - const: dphy-ref + - const: dphy-cfg + - const: grf + + '#phy-cells': + const: 0 + + power-domains: + description: Video in/out power domain. + maxItems: 1 + +required: + - compatible + - clocks + - clock-names + - '#phy-cells' + - power-domains + +additionalProperties: false + +examples: + - | + + /* + * MIPI D-PHY RX0 use registers in "general register files", it + * should be a child of the GRF. + * + * grf: syscon@ff770000 { + * compatible = "rockchip,rk3399-grf", "syscon", "simple-mfd"; + * ... + * }; + */ + + #include <dt-bindings/clock/rk3399-cru.h> + #include <dt-bindings/power/rk3399-power.h> + + mipi_dphy_rx0: mipi-dphy-rx0 { + compatible = "rockchip,rk3399-mipi-dphy-rx0"; + clocks = <&cru SCLK_MIPIDPHY_REF>, + <&cru SCLK_DPHY_RX0_CFG>, + <&cru PCLK_VIO_GRF>; + clock-names = "dphy-ref", "dphy-cfg", "grf"; + power-domains = <&power RK3399_PD_VIO>; + #phy-cells = <0>; + }; diff --git a/drivers/staging/media/phy-rockchip-dphy-rx0/Kconfig b/drivers/staging/media/phy-rockchip-dphy-rx0/Kconfig new file mode 100644 index 000000000000..bd0147624de1 --- /dev/null +++ b/drivers/staging/media/phy-rockchip-dphy-rx0/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config PHY_ROCKCHIP_DPHY_RX0 + tristate "Rockchip MIPI Synopsys DPHY RX0 driver" + depends on (ARCH_ROCKCHIP || COMPILE_TEST) && OF + select GENERIC_PHY_MIPI_DPHY + select GENERIC_PHY + help + Enable this to support the Rockchip MIPI Synopsys DPHY RX0 + associated to the Rockchip ISP module present in RK3399 SoCs. + + To compile this driver as a module, choose M here: the module + will be called phy-rockchip-dphy-rx0. diff --git a/drivers/staging/media/phy-rockchip-dphy-rx0/Makefile b/drivers/staging/media/phy-rockchip-dphy-rx0/Makefile new file mode 100644 index 000000000000..507e5d0593ab --- /dev/null +++ b/drivers/staging/media/phy-rockchip-dphy-rx0/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PHY_ROCKCHIP_DPHY_RX0) += phy-rockchip-dphy-rx0.o diff --git a/drivers/staging/media/phy-rockchip-dphy-rx0/TODO b/drivers/staging/media/phy-rockchip-dphy-rx0/TODO new file mode 100644 index 000000000000..ab612e5b27dc --- /dev/null +++ b/drivers/staging/media/phy-rockchip-dphy-rx0/TODO @@ -0,0 +1,6 @@ +The main reason for keeping this in staging is because the only driver +that uses this is rkisp1, which is also in staging. It should be moved together +with rkisp1. + +Please CC patches to Linux Media <linux-media@vger.kernel.org> and +Helen Koike <helen.koike@collabora.com>. diff --git a/drivers/staging/media/phy-rockchip-dphy-rx0/phy-rockchip-dphy-rx0.c b/drivers/staging/media/phy-rockchip-dphy-rx0/phy-rockchip-dphy-rx0.c new file mode 100644 index 000000000000..7c4df6d48c43 --- /dev/null +++ b/drivers/staging/media/phy-rockchip-dphy-rx0/phy-rockchip-dphy-rx0.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip MIPI Synopsys DPHY RX0 driver + * + * Copyright (C) 2019 Collabora, Ltd. + * + * Based on: + * + * drivers/media/platform/rockchip/isp1/mipi_dphy_sy.c + * in https://chromium.googlesource.com/chromiumos/third_party/kernel, + * chromeos-4.4 branch. + * + * Copyright (C) 2017 Fuzhou Rockchip Electronics Co., Ltd. + * Jacob Chen <jacob2.chen@rock-chips.com> + * Shunqian Zheng <zhengsq@rock-chips.com> + */ + +#include <linux/clk.h> +#include <linux/io.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/phy/phy.h> +#include <linux/phy/phy-mipi-dphy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#define RK3399_GRF_SOC_CON9 0x6224 +#define RK3399_GRF_SOC_CON21 0x6254 +#define RK3399_GRF_SOC_CON22 0x6258 +#define RK3399_GRF_SOC_CON23 0x625c +#define RK3399_GRF_SOC_CON24 0x6260 +#define RK3399_GRF_SOC_CON25 0x6264 +#define RK3399_GRF_SOC_STATUS1 0xe2a4 + +#define CLOCK_LANE_HS_RX_CONTROL 0x34 +#define LANE0_HS_RX_CONTROL 0x44 +#define LANE1_HS_RX_CONTROL 0x54 +#define LANE2_HS_RX_CONTROL 0x84 +#define LANE3_HS_RX_CONTROL 0x94 +#define LANES_THS_SETTLE_CONTROL 0x75 +#define THS_SETTLE_COUNTER_THRESHOLD 0x04 + +struct hsfreq_range { + u16 range_h; + u8 cfg_bit; +}; + +static const struct hsfreq_range rk3399_mipidphy_hsfreq_ranges[] = { + { 89, 0x00 }, { 99, 0x10 }, { 109, 0x20 }, { 129, 0x01 }, + { 139, 0x11 }, { 149, 0x21 }, { 169, 0x02 }, { 179, 0x12 }, + { 199, 0x22 }, { 219, 0x03 }, { 239, 0x13 }, { 249, 0x23 }, + { 269, 0x04 }, { 299, 0x14 }, { 329, 0x05 }, { 359, 0x15 }, + { 399, 0x25 }, { 449, 0x06 }, { 499, 0x16 }, { 549, 0x07 }, + { 599, 0x17 }, { 649, 0x08 }, { 699, 0x18 }, { 749, 0x09 }, + { 799, 0x19 }, { 849, 0x29 }, { 899, 0x39 }, { 949, 0x0a }, + { 999, 0x1a }, { 1049, 0x2a }, { 1099, 0x3a }, { 1149, 0x0b }, + { 1199, 0x1b }, { 1249, 0x2b }, { 1299, 0x3b }, { 1349, 0x0c }, + { 1399, 0x1c }, { 1449, 0x2c }, { 1500, 0x3c } +}; + +static const char * const rk3399_mipidphy_clks[] = { + "dphy-ref", + "dphy-cfg", + "grf", +}; + +enum dphy_reg_id { + GRF_DPHY_RX0_TURNDISABLE = 0, + GRF_DPHY_RX0_FORCERXMODE, + GRF_DPHY_RX0_FORCETXSTOPMODE, + GRF_DPHY_RX0_ENABLE, + GRF_DPHY_RX0_TESTCLR, + GRF_DPHY_RX0_TESTCLK, + GRF_DPHY_RX0_TESTEN, + GRF_DPHY_RX0_TESTDIN, + GRF_DPHY_RX0_TURNREQUEST, + GRF_DPHY_RX0_TESTDOUT, + GRF_DPHY_TX0_TURNDISABLE, + GRF_DPHY_TX0_FORCERXMODE, + GRF_DPHY_TX0_FORCETXSTOPMODE, + GRF_DPHY_TX0_TURNREQUEST, + GRF_DPHY_TX1RX1_TURNDISABLE, + GRF_DPHY_TX1RX1_FORCERXMODE, + GRF_DPHY_TX1RX1_FORCETXSTOPMODE, + GRF_DPHY_TX1RX1_ENABLE, + GRF_DPHY_TX1RX1_MASTERSLAVEZ, + GRF_DPHY_TX1RX1_BASEDIR, + GRF_DPHY_TX1RX1_ENABLECLK, + GRF_DPHY_TX1RX1_TURNREQUEST, + GRF_DPHY_RX1_SRC_SEL, + /* rk3288 only */ + GRF_CON_DISABLE_ISP, + GRF_CON_ISP_DPHY_SEL, + GRF_DSI_CSI_TESTBUS_SEL, + GRF_DVP_V18SEL, + /* below is for rk3399 only */ + GRF_DPHY_RX0_CLK_INV_SEL, + GRF_DPHY_RX1_CLK_INV_SEL, +}; + +struct dphy_reg { + u16 offset; + u8 mask; + u8 shift; +}; + +#define PHY_REG(_offset, _width, _shift) \ + { .offset = _offset, .mask = BIT(_width) - 1, .shift = _shift, } + +static const struct dphy_reg rk3399_grf_dphy_regs[] = { + [GRF_DPHY_RX0_TURNREQUEST] = PHY_REG(RK3399_GRF_SOC_CON9, 4, 0), + [GRF_DPHY_RX0_CLK_INV_SEL] = PHY_REG(RK3399_GRF_SOC_CON9, 1, 10), + [GRF_DPHY_RX1_CLK_INV_SEL] = PHY_REG(RK3399_GRF_SOC_CON9, 1, 11), + [GRF_DPHY_RX0_ENABLE] = PHY_REG(RK3399_GRF_SOC_CON21, 4, 0), + [GRF_DPHY_RX0_FORCERXMODE] = PHY_REG(RK3399_GRF_SOC_CON21, 4, 4), + [GRF_DPHY_RX0_FORCETXSTOPMODE] = PHY_REG(RK3399_GRF_SOC_CON21, 4, 8), + [GRF_DPHY_RX0_TURNDISABLE] = PHY_REG(RK3399_GRF_SOC_CON21, 4, 12), + [GRF_DPHY_TX0_FORCERXMODE] = PHY_REG(RK3399_GRF_SOC_CON22, 4, 0), + [GRF_DPHY_TX0_FORCETXSTOPMODE] = PHY_REG(RK3399_GRF_SOC_CON22, 4, 4), + [GRF_DPHY_TX0_TURNDISABLE] = PHY_REG(RK3399_GRF_SOC_CON22, 4, 8), + [GRF_DPHY_TX0_TURNREQUEST] = PHY_REG(RK3399_GRF_SOC_CON22, 4, 12), + [GRF_DPHY_TX1RX1_ENABLE] = PHY_REG(RK3399_GRF_SOC_CON23, 4, 0), + [GRF_DPHY_TX1RX1_FORCERXMODE] = PHY_REG(RK3399_GRF_SOC_CON23, 4, 4), + [GRF_DPHY_TX1RX1_FORCETXSTOPMODE] = PHY_REG(RK3399_GRF_SOC_CON23, 4, 8), + [GRF_DPHY_TX1RX1_TURNDISABLE] = PHY_REG(RK3399_GRF_SOC_CON23, 4, 12), + [GRF_DPHY_TX1RX1_TURNREQUEST] = PHY_REG(RK3399_GRF_SOC_CON24, 4, 0), + [GRF_DPHY_RX1_SRC_SEL] = PHY_REG(RK3399_GRF_SOC_CON24, 1, 4), + [GRF_DPHY_TX1RX1_BASEDIR] = PHY_REG(RK3399_GRF_SOC_CON24, 1, 5), + [GRF_DPHY_TX1RX1_ENABLECLK] = PHY_REG(RK3399_GRF_SOC_CON24, 1, 6), + [GRF_DPHY_TX1RX1_MASTERSLAVEZ] = PHY_REG(RK3399_GRF_SOC_CON24, 1, 7), + [GRF_DPHY_RX0_TESTDIN] = PHY_REG(RK3399_GRF_SOC_CON25, 8, 0), + [GRF_DPHY_RX0_TESTEN] = PHY_REG(RK3399_GRF_SOC_CON25, 1, 8), + [GRF_DPHY_RX0_TESTCLK] = PHY_REG(RK3399_GRF_SOC_CON25, 1, 9), + [GRF_DPHY_RX0_TESTCLR] = PHY_REG(RK3399_GRF_SOC_CON25, 1, 10), + [GRF_DPHY_RX0_TESTDOUT] = PHY_REG(RK3399_GRF_SOC_STATUS1, 8, 0), +}; + +struct rk_dphy_drv_data { + const char * const *clks; + unsigned int num_clks; + const struct hsfreq_range *hsfreq_ranges; + unsigned int num_hsfreq_ranges; + const struct dphy_reg *regs; +}; + +struct rk_dphy { + struct device *dev; + struct regmap *grf; + struct clk_bulk_data *clks; + + const struct rk_dphy_drv_data *drv_data; + struct phy_configure_opts_mipi_dphy config; + + u8 hsfreq; +}; + +static inline void rk_dphy_write_grf(struct rk_dphy *priv, + unsigned int index, u8 value) +{ + const struct dphy_reg *reg = &priv->drv_data->regs[index]; + /* Update high word */ + unsigned int val = (value << reg->shift) | + (reg->mask << (reg->shift + 16)); + + if (WARN_ON(!reg->offset)) + return; + regmap_write(priv->grf, reg->offset, val); +} + +static void rk_dphy_write(struct rk_dphy *priv, u8 test_code, u8 test_data) +{ + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTDIN, test_code); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTEN, 1); + /* + * With the falling edge on TESTCLK, the TESTDIN[7:0] signal content + * is latched internally as the current test code. Test data is + * programmed internally by rising edge on TESTCLK. + * This code assumes that TESTCLK is already 1. + */ + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTCLK, 0); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTEN, 0); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTDIN, test_data); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTCLK, 1); +} + +static void rk_dphy_enable(struct rk_dphy *priv) +{ + rk_dphy_write_grf(priv, GRF_DPHY_RX0_FORCERXMODE, 0); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_FORCETXSTOPMODE, 0); + + /* Disable lane turn around, which is ignored in receive mode */ + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TURNREQUEST, 0); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TURNDISABLE, 0xf); + + rk_dphy_write_grf(priv, GRF_DPHY_RX0_ENABLE, + GENMASK(priv->config.lanes - 1, 0)); + + /* dphy start */ + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTCLK, 1); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTCLR, 1); + usleep_range(100, 150); + rk_dphy_write_grf(priv, GRF_DPHY_RX0_TESTCLR, 0); + usleep_range(100, 150); + + /* set clock lane */ + /* HS hsfreq_range & lane 0 settle bypass */ + rk_dphy_write(priv, CLOCK_LANE_HS_RX_CONTROL, 0); + /* HS RX Control of lane0 */ + rk_dphy_write(priv, LANE0_HS_RX_CONTROL, priv->hsfreq << 1); + /* HS RX Control of lane1 */ + rk_dphy_write(priv, LANE1_HS_RX_CONTROL, priv->hsfreq << 1); + /* HS RX Control of lane2 */ + rk_dphy_write(priv, LANE2_HS_RX_CONTROL, priv->hsfreq << 1); + /* HS RX Control of lane3 */ + rk_dphy_write(priv, LANE3_HS_RX_CONTROL, priv->hsfreq << 1); + /* HS RX Data Lanes Settle State Time Control */ + rk_dphy_write(priv, LANES_THS_SETTLE_CONTROL, + THS_SETTLE_COUNTER_THRESHOLD); + + /* Normal operation */ + rk_dphy_write(priv, 0x0, 0); +} + +static int rk_dphy_configure(struct phy *phy, union phy_configure_opts *opts) +{ + struct rk_dphy *priv = phy_get_drvdata(phy); + const struct rk_dphy_drv_data *drv_data = priv->drv_data; + struct phy_configure_opts_mipi_dphy *config = &opts->mipi_dphy; + unsigned int hsfreq = 0; + unsigned int i; + u64 data_rate_mbps; + int ret; + + /* pass with phy_mipi_dphy_get_default_config (with pixel rate?) */ + ret = phy_mipi_dphy_config_validate(config); + if (ret) + return ret; + + data_rate_mbps = div_u64(config->hs_clk_rate, 1000 * 1000); + + dev_dbg(priv->dev, "lanes %d - data_rate_mbps %llu\n", + config->lanes, data_rate_mbps); + for (i = 0; i < drv_data->num_hsfreq_ranges; i++) { + if (drv_data->hsfreq_ranges[i].range_h >= data_rate_mbps) { + hsfreq = drv_data->hsfreq_ranges[i].cfg_bit; + break; + } + } + if (!hsfreq) + return -EINVAL; + + priv->hsfreq = hsfreq; + priv->config = *config; + return 0; +} + +static int rk_dphy_power_on(struct phy *phy) +{ + struct rk_dphy *priv = phy_get_drvdata(phy); + int ret; + + ret = clk_bulk_enable(priv->drv_data->num_clks, priv->clks); + if (ret) + return ret; + + rk_dphy_enable(priv); + + return 0; +} + +static int rk_dphy_power_off(struct phy *phy) +{ + struct rk_dphy *priv = phy_get_drvdata(phy); + + rk_dphy_write_grf(priv, GRF_DPHY_RX0_ENABLE, 0); + clk_bulk_disable(priv->drv_data->num_clks, priv->clks); + return 0; +} + +static int rk_dphy_init(struct phy *phy) +{ + struct rk_dphy *priv = phy_get_drvdata(phy); + + return clk_bulk_prepare(priv->drv_data->num_clks, priv->clks); +} + +static int rk_dphy_exit(struct phy *phy) +{ + struct rk_dphy *priv = phy_get_drvdata(phy); + + clk_bulk_unprepare(priv->drv_data->num_clks, priv->clks); + return 0; +} + +static const struct phy_ops rk_dphy_ops = { + .power_on = rk_dphy_power_on, + .power_off = rk_dphy_power_off, + .init = rk_dphy_init, + .exit = rk_dphy_exit, + .configure = rk_dphy_configure, + .owner = THIS_MODULE, +}; + +static const struct rk_dphy_drv_data rk3399_mipidphy_drv_data = { + .clks = rk3399_mipidphy_clks, + .num_clks = ARRAY_SIZE(rk3399_mipidphy_clks), + .hsfreq_ranges = rk3399_mipidphy_hsfreq_ranges, + .num_hsfreq_ranges = ARRAY_SIZE(rk3399_mipidphy_hsfreq_ranges), + .regs = rk3399_grf_dphy_regs, +}; + +static const struct of_device_id rk_dphy_dt_ids[] = { + { + .compatible = "rockchip,rk3399-mipi-dphy-rx0", + .data = &rk3399_mipidphy_drv_data, + }, + {} +}; +MODULE_DEVICE_TABLE(of, rk_dphy_dt_ids); + +static int rk_dphy_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + const struct rk_dphy_drv_data *drv_data; + struct phy_provider *phy_provider; + const struct of_device_id *of_id; + struct rk_dphy *priv; + struct phy *phy; + unsigned int i; + int ret; + + if (!dev->parent || !dev->parent->of_node) + return -ENODEV; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; + + priv->grf = syscon_node_to_regmap(dev->parent->of_node); + if (IS_ERR(priv->grf)) { + dev_err(dev, "Can't find GRF syscon\n"); + return -ENODEV; + } + + of_id = of_match_device(rk_dphy_dt_ids, dev); + if (!of_id) + return -EINVAL; + + drv_data = of_id->data; + priv->drv_data = drv_data; + priv->clks = devm_kcalloc(&pdev->dev, drv_data->num_clks, + sizeof(*priv->clks), GFP_KERNEL); + if (!priv->clks) + return -ENOMEM; + for (i = 0; i < drv_data->num_clks; i++) + priv->clks[i].id = drv_data->clks[i]; + ret = devm_clk_bulk_get(&pdev->dev, drv_data->num_clks, priv->clks); + if (ret) + return ret; + + phy = devm_phy_create(dev, np, &rk_dphy_ops); + if (IS_ERR(phy)) { + dev_err(dev, "failed to create phy\n"); + return PTR_ERR(phy); + } + phy_set_drvdata(phy, priv); + + phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); + + return PTR_ERR_OR_ZERO(phy_provider); +} + +static struct platform_driver rk_dphy_driver = { + .probe = rk_dphy_probe, + .driver = { + .name = "rockchip-mipi-dphy-rx0", + .of_match_table = rk_dphy_dt_ids, + }, +}; +module_platform_driver(rk_dphy_driver); + +MODULE_AUTHOR("Ezequiel Garcia <ezequiel@collabora.com>"); +MODULE_DESCRIPTION("Rockchip MIPI Synopsys DPHY RX0 driver"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/staging/media/rkisp1/Documentation/devicetree/bindings/media/rockchip-isp1.yaml b/drivers/staging/media/rkisp1/Documentation/devicetree/bindings/media/rockchip-isp1.yaml new file mode 100644 index 000000000000..af246b71eac6 --- /dev/null +++ b/drivers/staging/media/rkisp1/Documentation/devicetree/bindings/media/rockchip-isp1.yaml @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/rockchip-isp1.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip SoC Image Signal Processing unit v1 + +maintainers: + - Helen Koike <helen.koike@collabora.com> + +description: | + Rockchip ISP1 is the Camera interface for the Rockchip series of SoCs + which contains image processing, scaling, and compression functions. + +properties: + compatible: + const: rockchip,rk3399-cif-isp + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + phys: + maxItems: 1 + description: phandle for the PHY port + + phy-names: + const: dphy + + clocks: + items: + - description: ISP clock + - description: ISP AXI clock clock + - description: ISP AXI clock wrapper clock + - description: ISP AHB clock clock + - description: ISP AHB wrapper clock + + clock-names: + items: + - const: clk_isp + - const: aclk_isp + - const: aclk_isp_wrap + - const: hclk_isp + - const: hclk_isp_wrap + + # See ./video-interfaces.txt for details + ports: + type: object + additionalProperties: false + + properties: + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + port@0: + type: object + description: connection point for sensors at MIPI-DPHY RX0 + additionalProperties: false + + properties: + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + reg: + const: 0 + + patternProperties: + endpoint: + type: object + additionalProperties: false + + properties: + reg: + maxItems: 1 + + data-lanes: + minItems: 1 + maxItems: 4 + + remote-endpoint: true + + required: + - port@0 + +required: + - compatible + - interrupts + - clocks + - clock-names + - power-domains + - iommus + - phys + - phy-names + - ports + +additionalProperties: false + +examples: + - | + + #include <dt-bindings/clock/rk3399-cru.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/power/rk3399-power.h> + + parent0: parent@0 { + #address-cells = <2>; + #size-cells = <2>; + + isp0: isp0@ff910000 { + compatible = "rockchip,rk3399-cif-isp"; + reg = <0x0 0xff910000 0x0 0x4000>; + interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&cru SCLK_ISP0>, + <&cru ACLK_ISP0>, <&cru ACLK_ISP0_WRAPPER>, + <&cru HCLK_ISP0>, <&cru HCLK_ISP0_WRAPPER>; + clock-names = "clk_isp", + "aclk_isp", "aclk_isp_wrap", + "hclk_isp", "hclk_isp_wrap"; + power-domains = <&power RK3399_PD_ISP0>; + iommus = <&isp0_mmu>; + phys = <&dphy>; + phy-names = "dphy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + mipi_in_wcam: endpoint@0 { + reg = <0>; + remote-endpoint = <&wcam_out>; + data-lanes = <1 2>; + }; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1>; + }; + }; + }; + }; + + i2c7: i2c@ff160000 { + clock-frequency = <400000>; + #address-cells = <1>; + #size-cells = <0>; + + wcam: camera@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + + port { + wcam_out: endpoint { + remote-endpoint = <&mipi_in_wcam>; + data-lanes = <1 2>; + }; + }; + }; + + ucam: camera@3c { + compatible = "ovti,ov2685"; + reg = <0x3c>; + + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1>; + }; + }; + }; + }; + }; diff --git a/drivers/staging/media/rkisp1/Documentation/media/uapi/v4l/pixfmt-meta-rkisp1-params.rst b/drivers/staging/media/rkisp1/Documentation/media/uapi/v4l/pixfmt-meta-rkisp1-params.rst new file mode 100644 index 000000000000..32034e481357 --- /dev/null +++ b/drivers/staging/media/rkisp1/Documentation/media/uapi/v4l/pixfmt-meta-rkisp1-params.rst @@ -0,0 +1,23 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +.. _v4l2-meta-fmt-rkisp1-params: + +============================ +V4L2_META_FMT_RK_ISP1_PARAMS +============================ + +Rockchip ISP1 Parameters Data + +Description +=========== + +This format describes input parameters for the Rockchip ISP1. + +It uses c-struct :c:type:`rkisp1_params_cfg`, which is defined in +the ``linux/rkisp1-config.h`` header file. + +The parameters consist of multiple modules. +The module won't be updated if the corresponding bit was not set in module_*_update. + +.. kernel-doc:: include/uapi/linux/rkisp1-config.h + :functions: rkisp1_params_cfg diff --git a/drivers/staging/media/rkisp1/Documentation/media/uapi/v4l/pixfmt-meta-rkisp1-stat.rst b/drivers/staging/media/rkisp1/Documentation/media/uapi/v4l/pixfmt-meta-rkisp1-stat.rst new file mode 100644 index 000000000000..4ad303f96421 --- /dev/null +++ b/drivers/staging/media/rkisp1/Documentation/media/uapi/v4l/pixfmt-meta-rkisp1-stat.rst @@ -0,0 +1,22 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +.. _v4l2-meta-fmt-rkisp1-stat: + +============================= +V4L2_META_FMT_RK_ISP1_STAT_3A +============================= + + +Rockchip ISP1 Statistics Data + +Description +=========== + +This format describes image color statistics information generated by the Rockchip +ISP1. + +It uses c-struct :c:type:`rkisp1_stat_buffer`, which is defined in +the ``linux/rkisp1-config.h`` header file. + +.. kernel-doc:: include/uapi/linux/rkisp1-config.h + :functions: rkisp1_stat_buffer diff --git a/drivers/staging/media/rkisp1/Kconfig b/drivers/staging/media/rkisp1/Kconfig new file mode 100644 index 000000000000..b859a493caba --- /dev/null +++ b/drivers/staging/media/rkisp1/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_ROCKCHIP_ISP1 + tristate "Rockchip Image Signal Processing v1 Unit driver" + depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API + depends on ARCH_ROCKCHIP || COMPILE_TEST + select VIDEOBUF2_DMA_CONTIG + select VIDEOBUF2_VMALLOC + select V4L2_FWNODE + select PHY_ROCKCHIP_DPHY_RX0 + default n + help + Enable this to support the Image Signal Processing (ISP) module + present in RK3399 SoCs. + + To compile this driver as a module, choose M here: the module + will be called rockchip-isp1. diff --git a/drivers/staging/media/rkisp1/Makefile b/drivers/staging/media/rkisp1/Makefile new file mode 100644 index 000000000000..69ca59c7ef34 --- /dev/null +++ b/drivers/staging/media/rkisp1/Makefile @@ -0,0 +1,8 @@ +obj-$(CONFIG_VIDEO_ROCKCHIP_ISP1) += rockchip-isp1.o +rockchip-isp1-objs += rkisp1-capture.o \ + rkisp1-common.o \ + rkisp1-dev.o \ + rkisp1-isp.o \ + rkisp1-resizer.o \ + rkisp1-stats.o \ + rkisp1-params.o diff --git a/drivers/staging/media/rkisp1/TODO b/drivers/staging/media/rkisp1/TODO new file mode 100644 index 000000000000..03cd9a4e70f7 --- /dev/null +++ b/drivers/staging/media/rkisp1/TODO @@ -0,0 +1,23 @@ +* Fix serialization on subdev ops. +* Don't use v4l2_async_notifier_parse_fwnode_endpoints_by_port(). +e.g. isp_parse_of_endpoints in drivers/media/platform/omap3isp/isp.c +cio2_parse_firmware in drivers/media/pci/intel/ipu3/ipu3-cio2.c. +* Fix pad format size for statistics and parameters entities. +* Use threaded interrupt for rkisp1_stats_isr(), remove work queue. +* Fix checkpatch errors. +* Make sure uapi structs have the same size and layout in 32 and 62 bits, +and that there are no holes in the structures (pahole is a utility that +can be used to test this). +* Review and comment every lock +* Handle quantization +* Document rkisp1-common.h +* streaming paths (mainpath and selfpath) check if the other path is streaming +in several places of the code, review this, specially that it doesn't seem it +supports streaming from both paths at the same time. + +NOTES: +* All v4l2-compliance test must pass. +* Stats and params can be tested with libcamera and ChromiumOS stack. + +Please CC patches to Linux Media <linux-media@vger.kernel.org> and +Helen Koike <helen.koike@collabora.com>. diff --git a/drivers/staging/media/rkisp1/rkisp1-capture.c b/drivers/staging/media/rkisp1/rkisp1-capture.c new file mode 100644 index 000000000000..524e0dd38c1b --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-capture.c @@ -0,0 +1,1437 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip ISP1 Driver - V4l capture device + * + * Copyright (C) 2019 Collabora, Ltd. + * + * Based on Rockchip ISP1 driver by Rockchip Electronics Co., Ltd. + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#include <linux/delay.h> +#include <linux/pm_runtime.h> +#include <media/v4l2-common.h> +#include <media/v4l2-event.h> +#include <media/v4l2-fh.h> +#include <media/v4l2-ioctl.h> +#include <media/v4l2-mc.h> +#include <media/v4l2-subdev.h> +#include <media/videobuf2-dma-contig.h> + +#include "rkisp1-common.h" + +/* + * NOTE: There are two capture video devices in rkisp1, selfpath and mainpath. + * + * differences between selfpath and mainpath + * available mp sink input: isp + * available sp sink input : isp, dma(TODO) + * available mp sink pad fmts: yuv422, raw + * available sp sink pad fmts: yuv422, yuv420...... + * available mp source fmts: yuv, raw, jpeg(TODO) + * available sp source fmts: yuv, rgb + */ + +#define RKISP1_SP_DEV_NAME RKISP1_DRIVER_NAME "_selfpath" +#define RKISP1_MP_DEV_NAME RKISP1_DRIVER_NAME "_mainpath" + +#define RKISP1_MIN_BUFFERS_NEEDED 3 + +enum rkisp1_plane { + RKISP1_PLANE_Y = 0, + RKISP1_PLANE_CB = 1, + RKISP1_PLANE_CR = 2 +}; + +/* + * @fourcc: pixel format + * @fmt_type: helper filed for pixel format + * @uv_swap: if cb cr swaped, for yuv + * @write_format: defines how YCbCr self picture data is written to memory + * @output_format: defines sp output format + */ +struct rkisp1_capture_fmt_cfg { + u32 fourcc; + u8 fmt_type; + u8 uv_swap; + u32 write_format; + u32 output_format; +}; + +struct rkisp1_capture_ops { + void (*config)(struct rkisp1_capture *cap); + void (*stop)(struct rkisp1_capture *cap); + void (*enable)(struct rkisp1_capture *cap); + void (*disable)(struct rkisp1_capture *cap); + void (*set_data_path)(struct rkisp1_capture *cap); + bool (*is_stopped)(struct rkisp1_capture *cap); +}; + +struct rkisp1_capture_config { + const struct rkisp1_capture_fmt_cfg *fmts; + int fmt_size; + struct { + u32 y_size_init; + u32 cb_size_init; + u32 cr_size_init; + u32 y_base_ad_init; + u32 cb_base_ad_init; + u32 cr_base_ad_init; + u32 y_offs_cnt_init; + u32 cb_offs_cnt_init; + u32 cr_offs_cnt_init; + } mi; +}; + +static const struct rkisp1_capture_fmt_cfg rkisp1_mp_fmts[] = { + /* yuv422 */ + { + .fourcc = V4L2_PIX_FMT_YUYV, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT, + }, { + .fourcc = V4L2_PIX_FMT_YVYU, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT, + }, { + .fourcc = V4L2_PIX_FMT_VYUY, + .fmt_type = RKISP1_FMT_YUV, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT, + }, { + .fourcc = V4L2_PIX_FMT_YUV422P, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, { + .fourcc = V4L2_PIX_FMT_NV16, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA, + }, { + .fourcc = V4L2_PIX_FMT_NV61, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA, + }, { + .fourcc = V4L2_PIX_FMT_YVU422M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, + /* yuv420 */ + { + .fourcc = V4L2_PIX_FMT_NV21, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA, + }, { + .fourcc = V4L2_PIX_FMT_NV12, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA, + }, { + .fourcc = V4L2_PIX_FMT_NV21M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA, + }, { + .fourcc = V4L2_PIX_FMT_NV12M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA, + }, { + .fourcc = V4L2_PIX_FMT_YUV420, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, { + .fourcc = V4L2_PIX_FMT_YVU420, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, + /* yuv444 */ + { + .fourcc = V4L2_PIX_FMT_YUV444M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, + /* yuv400 */ + { + .fourcc = V4L2_PIX_FMT_GREY, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT, + }, + /* raw */ + { + .fourcc = V4L2_PIX_FMT_SRGGB8, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, { + .fourcc = V4L2_PIX_FMT_SGRBG8, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, { + .fourcc = V4L2_PIX_FMT_SGBRG8, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, { + .fourcc = V4L2_PIX_FMT_SBGGR8, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8, + }, { + .fourcc = V4L2_PIX_FMT_SRGGB10, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, { + .fourcc = V4L2_PIX_FMT_SGRBG10, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, { + .fourcc = V4L2_PIX_FMT_SGBRG10, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, { + .fourcc = V4L2_PIX_FMT_SBGGR10, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, { + .fourcc = V4L2_PIX_FMT_SRGGB12, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, { + .fourcc = V4L2_PIX_FMT_SGRBG12, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, { + .fourcc = V4L2_PIX_FMT_SGBRG12, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, { + .fourcc = V4L2_PIX_FMT_SBGGR12, + .fmt_type = RKISP1_FMT_BAYER, + .write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12, + }, +}; + +static const struct rkisp1_capture_fmt_cfg rkisp1_sp_fmts[] = { + /* yuv422 */ + { + .fourcc = V4L2_PIX_FMT_YUYV, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_INT, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422, + }, { + .fourcc = V4L2_PIX_FMT_YVYU, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_SP_WRITE_INT, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422, + }, { + .fourcc = V4L2_PIX_FMT_VYUY, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_SP_WRITE_INT, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422, + }, { + .fourcc = V4L2_PIX_FMT_YUV422P, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422, + }, { + .fourcc = V4L2_PIX_FMT_NV16, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422, + }, { + .fourcc = V4L2_PIX_FMT_NV61, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422, + }, { + .fourcc = V4L2_PIX_FMT_YVU422M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422, + }, + /* yuv420 */ + { + .fourcc = V4L2_PIX_FMT_NV21, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420, + }, { + .fourcc = V4L2_PIX_FMT_NV12, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420, + }, { + .fourcc = V4L2_PIX_FMT_NV21M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420, + }, { + .fourcc = V4L2_PIX_FMT_NV12M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420, + }, { + .fourcc = V4L2_PIX_FMT_YUV420, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420, + }, { + .fourcc = V4L2_PIX_FMT_YVU420, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 1, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420, + }, + /* yuv444 */ + { + .fourcc = V4L2_PIX_FMT_YUV444M, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV444, + }, + /* yuv400 */ + { + .fourcc = V4L2_PIX_FMT_GREY, + .fmt_type = RKISP1_FMT_YUV, + .uv_swap = 0, + .write_format = RKISP1_MI_CTRL_SP_WRITE_INT, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV400, + }, + /* rgb */ + { + .fourcc = V4L2_PIX_FMT_RGB24, + .fmt_type = RKISP1_FMT_RGB, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_RGB888, + }, { + .fourcc = V4L2_PIX_FMT_RGB565, + .fmt_type = RKISP1_FMT_RGB, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_RGB565, + }, { + .fourcc = V4L2_PIX_FMT_BGR666, + .fmt_type = RKISP1_FMT_RGB, + .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA, + .output_format = RKISP1_MI_CTRL_SP_OUTPUT_RGB666, + }, +}; + +static const struct rkisp1_capture_config rkisp1_capture_config_mp = { + .fmts = rkisp1_mp_fmts, + .fmt_size = ARRAY_SIZE(rkisp1_mp_fmts), + .mi = { + .y_size_init = RKISP1_CIF_MI_MP_Y_SIZE_INIT, + .cb_size_init = RKISP1_CIF_MI_MP_CB_SIZE_INIT, + .cr_size_init = RKISP1_CIF_MI_MP_CR_SIZE_INIT, + .y_base_ad_init = RKISP1_CIF_MI_MP_Y_BASE_AD_INIT, + .cb_base_ad_init = RKISP1_CIF_MI_MP_CB_BASE_AD_INIT, + .cr_base_ad_init = RKISP1_CIF_MI_MP_CR_BASE_AD_INIT, + .y_offs_cnt_init = RKISP1_CIF_MI_MP_Y_OFFS_CNT_INIT, + .cb_offs_cnt_init = RKISP1_CIF_MI_MP_CB_OFFS_CNT_INIT, + .cr_offs_cnt_init = RKISP1_CIF_MI_MP_CR_OFFS_CNT_INIT, + }, +}; + +static const struct rkisp1_capture_config rkisp1_capture_config_sp = { + .fmts = rkisp1_sp_fmts, + .fmt_size = ARRAY_SIZE(rkisp1_sp_fmts), + .mi = { + .y_size_init = RKISP1_CIF_MI_SP_Y_SIZE_INIT, + .cb_size_init = RKISP1_CIF_MI_SP_CB_SIZE_INIT, + .cr_size_init = RKISP1_CIF_MI_SP_CR_SIZE_INIT, + .y_base_ad_init = RKISP1_CIF_MI_SP_Y_BASE_AD_INIT, + .cb_base_ad_init = RKISP1_CIF_MI_SP_CB_BASE_AD_INIT, + .cr_base_ad_init = RKISP1_CIF_MI_SP_CR_BASE_AD_INIT, + .y_offs_cnt_init = RKISP1_CIF_MI_SP_Y_OFFS_CNT_INIT, + .cb_offs_cnt_init = RKISP1_CIF_MI_SP_CB_OFFS_CNT_INIT, + .cr_offs_cnt_init = RKISP1_CIF_MI_SP_CR_OFFS_CNT_INIT, + }, +}; + +static inline struct rkisp1_vdev_node * +rkisp1_vdev_to_node(struct video_device *vdev) +{ + return container_of(vdev, struct rkisp1_vdev_node, vdev); +} + +/* ---------------------------------------------------------------------------- + * Stream operations for self-picture path (sp) and main-picture path (mp) + */ + +static void rkisp1_mi_config_ctrl(struct rkisp1_capture *cap) +{ + u32 mi_ctrl = rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_CTRL); + + mi_ctrl &= ~GENMASK(17, 16); + mi_ctrl |= RKISP1_CIF_MI_CTRL_BURST_LEN_LUM_64; + + mi_ctrl &= ~GENMASK(19, 18); + mi_ctrl |= RKISP1_CIF_MI_CTRL_BURST_LEN_CHROM_64; + + mi_ctrl |= RKISP1_CIF_MI_CTRL_INIT_BASE_EN | + RKISP1_CIF_MI_CTRL_INIT_OFFSET_EN; + + rkisp1_write(cap->rkisp1, mi_ctrl, RKISP1_CIF_MI_CTRL); +} + +static u32 rkisp1_pixfmt_comp_size(const struct v4l2_pix_format_mplane *pixm, + unsigned int component) +{ + /* + * If packed format, then plane_fmt[0].sizeimage is the sum of all + * components, so we need to calculate just the size of Y component. + * See rkisp1_fill_pixfmt(). + */ + if (!component && pixm->num_planes == 1) + return pixm->plane_fmt[0].bytesperline * pixm->height; + return pixm->plane_fmt[component].sizeimage; +} + +static void rkisp1_irq_frame_end_enable(struct rkisp1_capture *cap) +{ + u32 mi_imsc = rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_IMSC); + + mi_imsc |= RKISP1_CIF_MI_FRAME(cap); + rkisp1_write(cap->rkisp1, mi_imsc, RKISP1_CIF_MI_IMSC); +} + +static void rkisp1_mp_config(struct rkisp1_capture *cap) +{ + const struct v4l2_pix_format_mplane *pixm = &cap->pix.fmt; + struct rkisp1_device *rkisp1 = cap->rkisp1; + u32 reg; + + rkisp1_write(rkisp1, rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_Y), + cap->config->mi.y_size_init); + rkisp1_write(rkisp1, rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_CB), + cap->config->mi.cb_size_init); + rkisp1_write(rkisp1, rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_CR), + cap->config->mi.cr_size_init); + + rkisp1_irq_frame_end_enable(cap); + if (cap->pix.cfg->uv_swap) { + reg = rkisp1_read(rkisp1, RKISP1_CIF_MI_XTD_FORMAT_CTRL); + + reg = (reg & ~BIT(0)) | + RKISP1_CIF_MI_XTD_FMT_CTRL_MP_CB_CR_SWAP; + rkisp1_write(rkisp1, reg, RKISP1_CIF_MI_XTD_FORMAT_CTRL); + } + + rkisp1_mi_config_ctrl(cap); + + reg = rkisp1_read(rkisp1, RKISP1_CIF_MI_CTRL); + reg &= ~RKISP1_MI_CTRL_MP_FMT_MASK; + reg |= cap->pix.cfg->write_format; + rkisp1_write(rkisp1, reg, RKISP1_CIF_MI_CTRL); + + reg = rkisp1_read(rkisp1, RKISP1_CIF_MI_CTRL); + reg |= RKISP1_CIF_MI_MP_AUTOUPDATE_ENABLE; + rkisp1_write(rkisp1, reg, RKISP1_CIF_MI_CTRL); +} + +static void rkisp1_sp_config(struct rkisp1_capture *cap) +{ + const struct v4l2_pix_format_mplane *pixm = &cap->pix.fmt; + struct rkisp1_device *rkisp1 = cap->rkisp1; + u32 mi_ctrl; + + rkisp1_write(rkisp1, rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_Y), + cap->config->mi.y_size_init); + rkisp1_write(rkisp1, rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_CB), + cap->config->mi.cb_size_init); + rkisp1_write(rkisp1, rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_CR), + cap->config->mi.cr_size_init); + + rkisp1_write(rkisp1, pixm->width, RKISP1_CIF_MI_SP_Y_PIC_WIDTH); + rkisp1_write(rkisp1, pixm->height, RKISP1_CIF_MI_SP_Y_PIC_HEIGHT); + rkisp1_write(rkisp1, cap->sp_y_stride, RKISP1_CIF_MI_SP_Y_LLENGTH); + + rkisp1_irq_frame_end_enable(cap); + if (cap->pix.cfg->uv_swap) { + u32 reg = rkisp1_read(rkisp1, RKISP1_CIF_MI_XTD_FORMAT_CTRL); + + rkisp1_write(rkisp1, reg & ~BIT(1), + RKISP1_CIF_MI_XTD_FORMAT_CTRL); + } + + rkisp1_mi_config_ctrl(cap); + + mi_ctrl = rkisp1_read(rkisp1, RKISP1_CIF_MI_CTRL); + mi_ctrl &= ~RKISP1_MI_CTRL_SP_FMT_MASK; + mi_ctrl |= cap->pix.cfg->write_format | + RKISP1_MI_CTRL_SP_INPUT_YUV422 | + cap->pix.cfg->output_format | + RKISP1_CIF_MI_SP_AUTOUPDATE_ENABLE; + rkisp1_write(rkisp1, mi_ctrl, RKISP1_CIF_MI_CTRL); +} + +static void rkisp1_mp_disable(struct rkisp1_capture *cap) +{ + u32 mi_ctrl = rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_CTRL); + + mi_ctrl &= ~(RKISP1_CIF_MI_CTRL_MP_ENABLE | + RKISP1_CIF_MI_CTRL_RAW_ENABLE); + rkisp1_write(cap->rkisp1, mi_ctrl, RKISP1_CIF_MI_CTRL); +} + +static void rkisp1_sp_disable(struct rkisp1_capture *cap) +{ + u32 mi_ctrl = rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_CTRL); + + mi_ctrl &= ~RKISP1_CIF_MI_CTRL_SP_ENABLE; + rkisp1_write(cap->rkisp1, mi_ctrl, RKISP1_CIF_MI_CTRL); +} + +static void rkisp1_mp_enable(struct rkisp1_capture *cap) +{ + const struct rkisp1_capture_fmt_cfg *isp_fmt = cap->pix.cfg; + u32 mi_ctrl; + + rkisp1_mp_disable(cap); + + mi_ctrl = rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_CTRL); + if (isp_fmt->fmt_type == RKISP1_FMT_BAYER) + mi_ctrl |= RKISP1_CIF_MI_CTRL_RAW_ENABLE; + /* YUV */ + else + mi_ctrl |= RKISP1_CIF_MI_CTRL_MP_ENABLE; + + rkisp1_write(cap->rkisp1, mi_ctrl, RKISP1_CIF_MI_CTRL); +} + +static void rkisp1_sp_enable(struct rkisp1_capture *cap) +{ + u32 mi_ctrl = rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_CTRL); + + mi_ctrl |= RKISP1_CIF_MI_CTRL_SP_ENABLE; + rkisp1_write(cap->rkisp1, mi_ctrl, RKISP1_CIF_MI_CTRL); +} + +static void rkisp1_mp_sp_stop(struct rkisp1_capture *cap) +{ + if (!cap->is_streaming) + return; + rkisp1_write(cap->rkisp1, + RKISP1_CIF_MI_FRAME(cap), RKISP1_CIF_MI_ICR); + cap->ops->disable(cap); +} + +static bool rkisp1_mp_is_stopped(struct rkisp1_capture *cap) +{ + u32 en = RKISP1_CIF_MI_CTRL_SHD_MP_IN_ENABLED | + RKISP1_CIF_MI_CTRL_SHD_RAW_OUT_ENABLED; + + return !(rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_CTRL_SHD) & en); +} + +static bool rkisp1_sp_is_stopped(struct rkisp1_capture *cap) +{ + return !(rkisp1_read(cap->rkisp1, RKISP1_CIF_MI_CTRL_SHD) & + RKISP1_CIF_MI_CTRL_SHD_SP_IN_ENABLED); +} + +static void rkisp1_mp_set_data_path(struct rkisp1_capture *cap) +{ + u32 dpcl = rkisp1_read(cap->rkisp1, RKISP1_CIF_VI_DPCL); + + dpcl = dpcl | RKISP1_CIF_VI_DPCL_CHAN_MODE_MP | + RKISP1_CIF_VI_DPCL_MP_MUX_MRSZ_MI; + rkisp1_write(cap->rkisp1, dpcl, RKISP1_CIF_VI_DPCL); +} + +static void rkisp1_sp_set_data_path(struct rkisp1_capture *cap) +{ + u32 dpcl = rkisp1_read(cap->rkisp1, RKISP1_CIF_VI_DPCL); + + dpcl |= RKISP1_CIF_VI_DPCL_CHAN_MODE_SP; + rkisp1_write(cap->rkisp1, dpcl, RKISP1_CIF_VI_DPCL); +} + +static struct rkisp1_capture_ops rkisp1_capture_ops_mp = { + .config = rkisp1_mp_config, + .enable = rkisp1_mp_enable, + .disable = rkisp1_mp_disable, + .stop = rkisp1_mp_sp_stop, + .set_data_path = rkisp1_mp_set_data_path, + .is_stopped = rkisp1_mp_is_stopped, +}; + +static struct rkisp1_capture_ops rkisp1_capture_ops_sp = { + .config = rkisp1_sp_config, + .enable = rkisp1_sp_enable, + .disable = rkisp1_sp_disable, + .stop = rkisp1_mp_sp_stop, + .set_data_path = rkisp1_sp_set_data_path, + .is_stopped = rkisp1_sp_is_stopped, +}; + +/* ---------------------------------------------------------------------------- + * Frame buffer operations + */ + +static int rkisp1_dummy_buf_create(struct rkisp1_capture *cap) +{ + const struct v4l2_pix_format_mplane *pixm = &cap->pix.fmt; + struct rkisp1_dummy_buffer *dummy_buf = &cap->buf.dummy; + + dummy_buf->size = max3(rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_Y), + rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_CB), + rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_CR)); + + /* The driver never access vaddr, no mapping is required */ + dummy_buf->vaddr = dma_alloc_attrs(cap->rkisp1->dev, + dummy_buf->size, + &dummy_buf->dma_addr, + GFP_KERNEL, + DMA_ATTR_NO_KERNEL_MAPPING); + if (!dummy_buf->vaddr) + return -ENOMEM; + + return 0; +} + +static void rkisp1_dummy_buf_destroy(struct rkisp1_capture *cap) +{ + dma_free_attrs(cap->rkisp1->dev, + cap->buf.dummy.size, cap->buf.dummy.vaddr, + cap->buf.dummy.dma_addr, DMA_ATTR_NO_KERNEL_MAPPING); +} + +static void rkisp1_set_next_buf(struct rkisp1_capture *cap) +{ + /* + * Use the dummy space allocated by dma_alloc_coherent to + * throw data if there is no available buffer. + */ + if (cap->buf.next) { + u32 *buff_addr = cap->buf.next->buff_addr; + + rkisp1_write(cap->rkisp1, + buff_addr[RKISP1_PLANE_Y], + cap->config->mi.y_base_ad_init); + rkisp1_write(cap->rkisp1, + buff_addr[RKISP1_PLANE_CB], + cap->config->mi.cb_base_ad_init); + rkisp1_write(cap->rkisp1, + buff_addr[RKISP1_PLANE_CR], + cap->config->mi.cr_base_ad_init); + } else { + rkisp1_write(cap->rkisp1, + cap->buf.dummy.dma_addr, + cap->config->mi.y_base_ad_init); + rkisp1_write(cap->rkisp1, + cap->buf.dummy.dma_addr, + cap->config->mi.cb_base_ad_init); + rkisp1_write(cap->rkisp1, + cap->buf.dummy.dma_addr, + cap->config->mi.cr_base_ad_init); + } + + /* Set plane offsets */ + rkisp1_write(cap->rkisp1, 0, cap->config->mi.y_offs_cnt_init); + rkisp1_write(cap->rkisp1, 0, cap->config->mi.cb_offs_cnt_init); + rkisp1_write(cap->rkisp1, 0, cap->config->mi.cr_offs_cnt_init); +} + +/* + * This function is called when a frame end comes. The next frame + * is processing and we should set up buffer for next-next frame, + * otherwise it will overflow. + */ +static void rkisp1_handle_buffer(struct rkisp1_capture *cap) +{ + struct rkisp1_isp *isp = &cap->rkisp1->isp; + struct rkisp1_buffer *curr_buf = cap->buf.curr; + unsigned long flags; + + spin_lock_irqsave(&cap->buf.lock, flags); + + if (curr_buf) { + curr_buf->vb.sequence = atomic_read(&isp->frame_sequence); + curr_buf->vb.vb2_buf.timestamp = ktime_get_boottime_ns(); + curr_buf->vb.field = V4L2_FIELD_NONE; + vb2_buffer_done(&curr_buf->vb.vb2_buf, VB2_BUF_STATE_DONE); + } else { + cap->rkisp1->debug.frame_drop[cap->id]++; + } + + cap->buf.curr = cap->buf.next; + cap->buf.next = NULL; + + if (!list_empty(&cap->buf.queue)) { + cap->buf.next = list_first_entry(&cap->buf.queue, + struct rkisp1_buffer, + queue); + list_del(&cap->buf.next->queue); + } + spin_unlock_irqrestore(&cap->buf.lock, flags); + + rkisp1_set_next_buf(cap); +} + +void rkisp1_capture_isr(struct rkisp1_device *rkisp1) +{ + unsigned int i; + u32 status; + + status = rkisp1_read(rkisp1, RKISP1_CIF_MI_MIS); + rkisp1_write(rkisp1, status, RKISP1_CIF_MI_ICR); + + for (i = 0; i < ARRAY_SIZE(rkisp1->capture_devs); ++i) { + struct rkisp1_capture *cap = &rkisp1->capture_devs[i]; + + if (!(status & RKISP1_CIF_MI_FRAME(cap))) + continue; + if (!cap->is_stopping) { + rkisp1_handle_buffer(cap); + continue; + } + /* + * Make sure stream is actually stopped, whose state + * can be read from the shadow register, before + * wake_up() thread which would immediately free all + * frame buffers. stop() takes effect at the next + * frame end that sync the configurations to shadow + * regs. + */ + if (!cap->ops->is_stopped(cap)) { + cap->ops->stop(cap); + continue; + } + cap->is_stopping = false; + cap->is_streaming = false; + wake_up(&cap->done); + } +} + +/* ---------------------------------------------------------------------------- + * Vb2 operations + */ + +static int rkisp1_vb2_queue_setup(struct vb2_queue *queue, + unsigned int *num_buffers, + unsigned int *num_planes, + unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct rkisp1_capture *cap = queue->drv_priv; + const struct v4l2_pix_format_mplane *pixm = &cap->pix.fmt; + unsigned int i; + + if (*num_planes) { + if (*num_planes != pixm->num_planes) + return -EINVAL; + + for (i = 0; i < pixm->num_planes; i++) + if (sizes[i] < pixm->plane_fmt[i].sizeimage) + return -EINVAL; + } else { + *num_planes = pixm->num_planes; + for (i = 0; i < pixm->num_planes; i++) + sizes[i] = pixm->plane_fmt[i].sizeimage; + } + + return 0; +} + +static void rkisp1_vb2_buf_queue(struct vb2_buffer *vb) +{ + struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); + struct rkisp1_buffer *ispbuf = + container_of(vbuf, struct rkisp1_buffer, vb); + struct rkisp1_capture *cap = vb->vb2_queue->drv_priv; + const struct v4l2_pix_format_mplane *pixm = &cap->pix.fmt; + unsigned long flags; + unsigned int i; + + memset(ispbuf->buff_addr, 0, sizeof(ispbuf->buff_addr)); + for (i = 0; i < pixm->num_planes; i++) + ispbuf->buff_addr[i] = vb2_dma_contig_plane_dma_addr(vb, i); + + /* Convert to non-MPLANE */ + if (pixm->num_planes == 1) { + ispbuf->buff_addr[RKISP1_PLANE_CB] = + ispbuf->buff_addr[RKISP1_PLANE_Y] + + rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_Y); + ispbuf->buff_addr[RKISP1_PLANE_CR] = + ispbuf->buff_addr[RKISP1_PLANE_CB] + + rkisp1_pixfmt_comp_size(pixm, RKISP1_PLANE_CB); + } + + spin_lock_irqsave(&cap->buf.lock, flags); + + /* + * If there's no next buffer assigned, queue this buffer directly + * as the next buffer, and update the memory interface. + */ + if (cap->is_streaming && !cap->buf.next && + atomic_read(&cap->rkisp1->isp.frame_sequence) == -1) { + cap->buf.next = ispbuf; + rkisp1_set_next_buf(cap); + } else { + list_add_tail(&ispbuf->queue, &cap->buf.queue); + } + spin_unlock_irqrestore(&cap->buf.lock, flags); +} + +static int rkisp1_vb2_buf_prepare(struct vb2_buffer *vb) +{ + struct rkisp1_capture *cap = vb->vb2_queue->drv_priv; + unsigned int i; + + for (i = 0; i < cap->pix.fmt.num_planes; i++) { + unsigned long size = cap->pix.fmt.plane_fmt[i].sizeimage; + + if (vb2_plane_size(vb, i) < size) { + dev_err(cap->rkisp1->dev, + "User buffer too small (%ld < %ld)\n", + vb2_plane_size(vb, i), size); + return -EINVAL; + } + vb2_set_plane_payload(vb, i, size); + } + + return 0; +} + +static void rkisp1_return_all_buffers(struct rkisp1_capture *cap, + enum vb2_buffer_state state) +{ + unsigned long flags; + struct rkisp1_buffer *buf; + + spin_lock_irqsave(&cap->buf.lock, flags); + if (cap->buf.curr) { + vb2_buffer_done(&cap->buf.curr->vb.vb2_buf, state); + cap->buf.curr = NULL; + } + if (cap->buf.next) { + vb2_buffer_done(&cap->buf.next->vb.vb2_buf, state); + cap->buf.next = NULL; + } + while (!list_empty(&cap->buf.queue)) { + buf = list_first_entry(&cap->buf.queue, + struct rkisp1_buffer, queue); + list_del(&buf->queue); + vb2_buffer_done(&buf->vb.vb2_buf, state); + } + spin_unlock_irqrestore(&cap->buf.lock, flags); +} + +/* + * rkisp1_pipeline_sink_walk - Walk through the pipeline and call cb + * @from: entity at which to start pipeline walk + * @until: entity at which to stop pipeline walk + * + * Walk the entities chain starting at the pipeline video node and stop + * all subdevices in the chain. + * + * If the until argument isn't NULL, stop the pipeline walk when reaching the + * until entity. This is used to disable a partially started pipeline due to a + * subdev start error. + */ +static int rkisp1_pipeline_sink_walk(struct media_entity *from, + struct media_entity *until, + int (*cb)(struct media_entity *from, + struct media_entity *curr)) +{ + struct media_entity *entity = from; + struct media_pad *pad; + unsigned int i; + int ret; + + while (1) { + pad = NULL; + /* Find remote source pad */ + for (i = 0; i < entity->num_pads; i++) { + struct media_pad *spad = &entity->pads[i]; + + if (!(spad->flags & MEDIA_PAD_FL_SINK)) + continue; + pad = media_entity_remote_pad(spad); + if (pad && is_media_entity_v4l2_subdev(pad->entity)) + break; + } + if (!pad || !is_media_entity_v4l2_subdev(pad->entity)) + break; + + entity = pad->entity; + if (entity == until) + break; + + ret = cb(from, entity); + if (ret) + return ret; + } + + return 0; +} + +static int rkisp1_pipeline_disable_cb(struct media_entity *from, + struct media_entity *curr) +{ + struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(curr); + + return v4l2_subdev_call(sd, video, s_stream, false); +} + +static int rkisp1_pipeline_enable_cb(struct media_entity *from, + struct media_entity *curr) +{ + struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(curr); + + return v4l2_subdev_call(sd, video, s_stream, true); +} + +static void rkisp1_stream_stop(struct rkisp1_capture *cap) +{ + int ret; + + /* Stream should stop in interrupt. If it dosn't, stop it by force. */ + cap->is_stopping = true; + ret = wait_event_timeout(cap->done, + !cap->is_streaming, + msecs_to_jiffies(1000)); + if (!ret) { + cap->rkisp1->debug.stop_timeout[cap->id]++; + cap->ops->stop(cap); + cap->is_stopping = false; + cap->is_streaming = false; + } +} + +static void rkisp1_vb2_stop_streaming(struct vb2_queue *queue) +{ + struct rkisp1_capture *cap = queue->drv_priv; + struct rkisp1_vdev_node *node = &cap->vnode; + struct rkisp1_device *rkisp1 = cap->rkisp1; + int ret; + + rkisp1_stream_stop(cap); + media_pipeline_stop(&node->vdev.entity); + ret = rkisp1_pipeline_sink_walk(&node->vdev.entity, NULL, + rkisp1_pipeline_disable_cb); + if (ret) + dev_err(rkisp1->dev, + "pipeline stream-off failed error:%d\n", ret); + + rkisp1_return_all_buffers(cap, VB2_BUF_STATE_ERROR); + + ret = v4l2_pipeline_pm_use(&node->vdev.entity, 0); + if (ret) + dev_err(rkisp1->dev, "pipeline close failed error:%d\n", ret); + + ret = pm_runtime_put(rkisp1->dev); + if (ret) + dev_err(rkisp1->dev, "power down failed error:%d\n", ret); + + rkisp1_dummy_buf_destroy(cap); +} + +/* + * Most of registers inside rockchip ISP1 have shadow register since + * they must be not be changed during processing a frame. + * Usually, each sub-module updates its shadow register after + * processing the last pixel of a frame. + */ +static void rkisp1_stream_start(struct rkisp1_capture *cap) +{ + struct rkisp1_device *rkisp1 = cap->rkisp1; + struct rkisp1_capture *other = &rkisp1->capture_devs[cap->id ^ 1]; + + cap->ops->set_data_path(cap); + cap->ops->config(cap); + + /* Setup a buffer for the next frame */ + rkisp1_handle_buffer(cap); + cap->ops->enable(cap); + /* It's safe to config ACTIVE and SHADOW regs for the + * first stream. While when the second is starting, do NOT + * force update because it also update the first one. + * + * The latter case would drop one more buf(that is 2) since + * there's not buf in shadow when the second FE received. This's + * also required because the second FE maybe corrupt especially + * when run at 120fps. + */ + if (!other->is_streaming) { + /* force cfg update */ + rkisp1_write(rkisp1, + RKISP1_CIF_MI_INIT_SOFT_UPD, RKISP1_CIF_MI_INIT); + rkisp1_handle_buffer(cap); + } + cap->is_streaming = true; +} + +static int +rkisp1_vb2_start_streaming(struct vb2_queue *queue, unsigned int count) +{ + struct rkisp1_capture *cap = queue->drv_priv; + struct media_entity *entity = &cap->vnode.vdev.entity; + int ret; + + ret = rkisp1_dummy_buf_create(cap); + if (ret) + goto err_ret_buffers; + + ret = pm_runtime_get_sync(cap->rkisp1->dev); + if (ret) { + dev_err(cap->rkisp1->dev, "power up failed %d\n", ret); + goto err_destroy_dummy; + } + ret = v4l2_pipeline_pm_use(entity, 1); + if (ret) { + dev_err(cap->rkisp1->dev, "open cif pipeline failed %d\n", ret); + goto err_pipe_pm_put; + } + + rkisp1_stream_start(cap); + + /* start sub-devices */ + ret = rkisp1_pipeline_sink_walk(entity, NULL, + rkisp1_pipeline_enable_cb); + if (ret) + goto err_stop_stream; + + ret = media_pipeline_start(entity, &cap->rkisp1->pipe); + if (ret) { + dev_err(cap->rkisp1->dev, "start pipeline failed %d\n", ret); + goto err_pipe_disable; + } + + return 0; + +err_pipe_disable: + rkisp1_pipeline_sink_walk(entity, NULL, rkisp1_pipeline_disable_cb); +err_stop_stream: + rkisp1_stream_stop(cap); + v4l2_pipeline_pm_use(entity, 0); +err_pipe_pm_put: + pm_runtime_put(cap->rkisp1->dev); +err_destroy_dummy: + rkisp1_dummy_buf_destroy(cap); +err_ret_buffers: + rkisp1_return_all_buffers(cap, VB2_BUF_STATE_QUEUED); + + return ret; +} + +static struct vb2_ops rkisp1_vb2_ops = { + .queue_setup = rkisp1_vb2_queue_setup, + .buf_queue = rkisp1_vb2_buf_queue, + .buf_prepare = rkisp1_vb2_buf_prepare, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, + .stop_streaming = rkisp1_vb2_stop_streaming, + .start_streaming = rkisp1_vb2_start_streaming, +}; + +/* ---------------------------------------------------------------------------- + * IOCTLs operations + */ + +static const struct v4l2_format_info * +rkisp1_fill_pixfmt(struct v4l2_pix_format_mplane *pixm, + enum rkisp1_stream_id id) +{ + struct v4l2_plane_pix_format *plane_y = &pixm->plane_fmt[0]; + const struct v4l2_format_info *info; + unsigned int i; + u32 stride; + + info = v4l2_format_info(pixm->pixelformat); + pixm->num_planes = info->mem_planes; + stride = info->bpp[0] * pixm->width; + /* Self path supports custom stride but Main path doesn't */ + if (id == RKISP1_MAINPATH || plane_y->bytesperline < stride) + plane_y->bytesperline = stride; + plane_y->sizeimage = plane_y->bytesperline * pixm->height; + + /* normalize stride to pixels per line */ + stride = DIV_ROUND_UP(plane_y->bytesperline, info->bpp[0]); + + for (i = 1; i < info->comp_planes; i++) { + struct v4l2_plane_pix_format *plane = &pixm->plane_fmt[i]; + + /* bytesperline for other components derive from Y component */ + plane->bytesperline = DIV_ROUND_UP(stride, info->hdiv) * + info->bpp[i]; + plane->sizeimage = plane->bytesperline * + DIV_ROUND_UP(pixm->height, info->vdiv); + } + + /* + * If pixfmt is packed, then plane_fmt[0] should contain the total size + * considering all components. plane_fmt[i] for i > 0 should be ignored + * by userspace as mem_planes == 1, but we are keeping information there + * for convenience. + */ + if (info->mem_planes == 1) + for (i = 1; i < info->comp_planes; i++) + plane_y->sizeimage += pixm->plane_fmt[i].sizeimage; + + return info; +} + +static const struct rkisp1_capture_fmt_cfg * +rkisp1_find_fmt_cfg(const struct rkisp1_capture *cap, const u32 pixelfmt) +{ + unsigned int i; + + for (i = 0; i < cap->config->fmt_size; i++) { + if (cap->config->fmts[i].fourcc == pixelfmt) + return &cap->config->fmts[i]; + } + return NULL; +} + +static void rkisp1_try_fmt(const struct rkisp1_capture *cap, + struct v4l2_pix_format_mplane *pixm, + const struct rkisp1_capture_fmt_cfg **fmt_cfg, + const struct v4l2_format_info **fmt_info) +{ + const struct rkisp1_capture_config *config = cap->config; + struct rkisp1_capture *other_cap = + &cap->rkisp1->capture_devs[cap->id ^ 1]; + const struct rkisp1_capture_fmt_cfg *fmt; + const struct v4l2_format_info *info; + const unsigned int max_widths[] = { RKISP1_RSZ_MP_SRC_MAX_WIDTH, + RKISP1_RSZ_SP_SRC_MAX_WIDTH }; + const unsigned int max_heights[] = { RKISP1_RSZ_MP_SRC_MAX_HEIGHT, + RKISP1_RSZ_SP_SRC_MAX_HEIGHT}; + + fmt = rkisp1_find_fmt_cfg(cap, pixm->pixelformat); + if (!fmt) { + fmt = config->fmts; + pixm->pixelformat = fmt->fourcc; + } + + pixm->width = clamp_t(u32, pixm->width, + RKISP1_RSZ_SRC_MIN_WIDTH, max_widths[cap->id]); + pixm->height = clamp_t(u32, pixm->height, + RKISP1_RSZ_SRC_MIN_HEIGHT, max_heights[cap->id]); + + pixm->field = V4L2_FIELD_NONE; + pixm->colorspace = V4L2_COLORSPACE_DEFAULT; + pixm->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + + info = rkisp1_fill_pixfmt(pixm, cap->id); + + /* can not change quantization when stream-on */ + if (other_cap->is_streaming) + pixm->quantization = other_cap->pix.fmt.quantization; + /* output full range by default, take effect in params */ + else if (!pixm->quantization || + pixm->quantization > V4L2_QUANTIZATION_LIM_RANGE) + pixm->quantization = V4L2_QUANTIZATION_FULL_RANGE; + + if (fmt_cfg) + *fmt_cfg = fmt; + if (fmt_info) + *fmt_info = info; +} + +static void rkisp1_set_fmt(struct rkisp1_capture *cap, + struct v4l2_pix_format_mplane *pixm) +{ + rkisp1_try_fmt(cap, pixm, &cap->pix.cfg, &cap->pix.info); + cap->pix.fmt = *pixm; + + /* SP supports custom stride in number of pixels of the Y plane */ + if (cap->id == RKISP1_SELFPATH) + cap->sp_y_stride = pixm->plane_fmt[0].bytesperline / + cap->pix.info->bpp[0]; +} + +static int rkisp1_try_fmt_vid_cap_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct rkisp1_capture *cap = video_drvdata(file); + + rkisp1_try_fmt(cap, &f->fmt.pix_mp, NULL, NULL); + + return 0; +} + +static int rkisp1_enum_fmt_vid_cap_mplane(struct file *file, void *priv, + struct v4l2_fmtdesc *f) +{ + struct rkisp1_capture *cap = video_drvdata(file); + const struct rkisp1_capture_fmt_cfg *fmt = NULL; + + if (f->index >= cap->config->fmt_size) + return -EINVAL; + + fmt = &cap->config->fmts[f->index]; + f->pixelformat = fmt->fourcc; + + return 0; +} + +static int rkisp1_s_fmt_vid_cap_mplane(struct file *file, + void *priv, struct v4l2_format *f) +{ + struct rkisp1_capture *cap = video_drvdata(file); + struct rkisp1_vdev_node *node = + rkisp1_vdev_to_node(&cap->vnode.vdev); + + if (vb2_is_busy(&node->buf_queue)) + return -EBUSY; + + rkisp1_set_fmt(cap, &f->fmt.pix_mp); + + return 0; +} + +static int rkisp1_g_fmt_vid_cap_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct rkisp1_capture *cap = video_drvdata(file); + + f->fmt.pix_mp = cap->pix.fmt; + + return 0; +} + +static int +rkisp1_querycap(struct file *file, void *priv, struct v4l2_capability *cap) +{ + struct rkisp1_capture *cap_dev = video_drvdata(file); + struct rkisp1_device *rkisp1 = cap_dev->rkisp1; + + strscpy(cap->driver, rkisp1->dev->driver->name, sizeof(cap->driver)); + strscpy(cap->card, rkisp1->dev->driver->name, sizeof(cap->card)); + strscpy(cap->bus_info, RKISP1_BUS_INFO, sizeof(cap->bus_info)); + + return 0; +} + +static const struct v4l2_ioctl_ops rkisp1_v4l2_ioctl_ops = { + .vidioc_reqbufs = vb2_ioctl_reqbufs, + .vidioc_querybuf = vb2_ioctl_querybuf, + .vidioc_create_bufs = vb2_ioctl_create_bufs, + .vidioc_qbuf = vb2_ioctl_qbuf, + .vidioc_expbuf = vb2_ioctl_expbuf, + .vidioc_dqbuf = vb2_ioctl_dqbuf, + .vidioc_prepare_buf = vb2_ioctl_prepare_buf, + .vidioc_streamon = vb2_ioctl_streamon, + .vidioc_streamoff = vb2_ioctl_streamoff, + .vidioc_try_fmt_vid_cap_mplane = rkisp1_try_fmt_vid_cap_mplane, + .vidioc_s_fmt_vid_cap_mplane = rkisp1_s_fmt_vid_cap_mplane, + .vidioc_g_fmt_vid_cap_mplane = rkisp1_g_fmt_vid_cap_mplane, + .vidioc_enum_fmt_vid_cap = rkisp1_enum_fmt_vid_cap_mplane, + .vidioc_querycap = rkisp1_querycap, + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +static int rkisp1_capture_link_validate(struct media_link *link) +{ + struct video_device *vdev = + media_entity_to_video_device(link->sink->entity); + struct v4l2_subdev *sd = + media_entity_to_v4l2_subdev(link->source->entity); + struct rkisp1_capture *cap = video_get_drvdata(vdev); + struct rkisp1_isp *isp = &cap->rkisp1->isp; + struct v4l2_subdev_format sd_fmt; + int ret; + + if (cap->id == RKISP1_SELFPATH && + isp->src_fmt->mbus_code != MEDIA_BUS_FMT_YUYV8_2X8) { + dev_err(cap->rkisp1->dev, + "selfpath only supports MEDIA_BUS_FMT_YUYV8_2X8\n"); + return -EPIPE; + } + + if (cap->pix.cfg->fmt_type != isp->src_fmt->fmt_type) { + dev_err(cap->rkisp1->dev, + "format type mismatch in link '%s:%d->%s:%d'\n", + link->source->entity->name, link->source->index, + link->sink->entity->name, link->sink->index); + return -EPIPE; + } + + sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE; + sd_fmt.pad = link->source->index; + ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &sd_fmt); + if (ret) + return ret; + + if (sd_fmt.format.height != cap->pix.fmt.height || + sd_fmt.format.width != cap->pix.fmt.width) + return -EPIPE; + + return 0; +} + +/* ---------------------------------------------------------------------------- + * core functions + */ + +static const struct media_entity_operations rkisp1_media_ops = { + .link_validate = rkisp1_capture_link_validate, +}; + +static const struct v4l2_file_operations rkisp1_fops = { + .open = v4l2_fh_open, + .release = vb2_fop_release, + .unlocked_ioctl = video_ioctl2, + .poll = vb2_fop_poll, + .mmap = vb2_fop_mmap, +}; + +static void rkisp1_unregister_capture(struct rkisp1_capture *cap) +{ + media_entity_cleanup(&cap->vnode.vdev.entity); + video_unregister_device(&cap->vnode.vdev); +} + +void rkisp1_capture_devs_unregister(struct rkisp1_device *rkisp1) +{ + struct rkisp1_capture *mp = &rkisp1->capture_devs[RKISP1_MAINPATH]; + struct rkisp1_capture *sp = &rkisp1->capture_devs[RKISP1_SELFPATH]; + + rkisp1_unregister_capture(mp); + rkisp1_unregister_capture(sp); +} + +static int rkisp1_register_capture(struct rkisp1_capture *cap) +{ + const char * const dev_names[] = {RKISP1_MP_DEV_NAME, + RKISP1_SP_DEV_NAME}; + struct v4l2_device *v4l2_dev = &cap->rkisp1->v4l2_dev; + struct video_device *vdev = &cap->vnode.vdev; + struct rkisp1_vdev_node *node; + struct vb2_queue *q; + int ret; + + strscpy(vdev->name, dev_names[cap->id], sizeof(vdev->name)); + node = rkisp1_vdev_to_node(vdev); + mutex_init(&node->vlock); + + vdev->ioctl_ops = &rkisp1_v4l2_ioctl_ops; + vdev->release = video_device_release_empty; + vdev->fops = &rkisp1_fops; + vdev->minor = -1; + vdev->v4l2_dev = v4l2_dev; + vdev->lock = &node->vlock; + vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE | + V4L2_CAP_STREAMING; + vdev->entity.ops = &rkisp1_media_ops; + video_set_drvdata(vdev, cap); + vdev->vfl_dir = VFL_DIR_RX; + node->pad.flags = MEDIA_PAD_FL_SINK; + + q = &node->buf_queue; + q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + q->io_modes = VB2_MMAP | VB2_DMABUF | VB2_USERPTR; + q->drv_priv = cap; + q->ops = &rkisp1_vb2_ops; + q->mem_ops = &vb2_dma_contig_memops; + q->buf_struct_size = sizeof(struct rkisp1_buffer); + q->min_buffers_needed = RKISP1_MIN_BUFFERS_NEEDED; + q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + q->lock = &node->vlock; + q->dev = cap->rkisp1->dev; + ret = vb2_queue_init(q); + if (ret) { + dev_err(cap->rkisp1->dev, + "vb2 queue init failed (err=%d)\n", ret); + return ret; + } + + vdev->queue = q; + + ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1); + if (ret) { + dev_err(cap->rkisp1->dev, + "failed to register %s, ret=%d\n", vdev->name, ret); + return ret; + } + v4l2_info(v4l2_dev, "registered %s as /dev/video%d\n", vdev->name, + vdev->num); + + ret = media_entity_pads_init(&vdev->entity, 1, &node->pad); + if (ret) { + video_unregister_device(vdev); + return ret; + } + + return 0; +} + +static void +rkisp1_capture_init(struct rkisp1_device *rkisp1, enum rkisp1_stream_id id) +{ + struct rkisp1_capture *cap = &rkisp1->capture_devs[id]; + struct v4l2_pix_format_mplane pixm; + + memset(cap, 0, sizeof(*cap)); + cap->id = id; + cap->rkisp1 = rkisp1; + + INIT_LIST_HEAD(&cap->buf.queue); + init_waitqueue_head(&cap->done); + spin_lock_init(&cap->buf.lock); + if (cap->id == RKISP1_SELFPATH) { + cap->ops = &rkisp1_capture_ops_sp; + cap->config = &rkisp1_capture_config_sp; + } else { + cap->ops = &rkisp1_capture_ops_mp; + cap->config = &rkisp1_capture_config_mp; + } + + cap->is_streaming = false; + + memset(&pixm, 0, sizeof(pixm)); + pixm.pixelformat = V4L2_PIX_FMT_YUYV; + pixm.width = RKISP1_DEFAULT_WIDTH; + pixm.height = RKISP1_DEFAULT_HEIGHT; + rkisp1_set_fmt(cap, &pixm); +} + +int rkisp1_capture_devs_register(struct rkisp1_device *rkisp1) +{ + struct rkisp1_capture *cap; + unsigned int i, j; + int ret; + + for (i = 0; i < ARRAY_SIZE(rkisp1->capture_devs); i++) { + rkisp1_capture_init(rkisp1, i); + cap = &rkisp1->capture_devs[i]; + cap->rkisp1 = rkisp1; + ret = rkisp1_register_capture(cap); + if (ret) + goto err_unreg_capture_devs; + } + + return 0; + +err_unreg_capture_devs: + for (j = 0; j < i; j++) { + cap = &rkisp1->capture_devs[j]; + rkisp1_unregister_capture(cap); + } + + return ret; +} diff --git a/drivers/staging/media/rkisp1/rkisp1-common.c b/drivers/staging/media/rkisp1/rkisp1-common.c new file mode 100644 index 000000000000..cf889666e166 --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-common.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip ISP1 Driver - Common definitions + * + * Copyright (C) 2019 Collabora, Ltd. + */ + +#include <media/v4l2-rect.h> + +#include "rkisp1-common.h" + +static const struct v4l2_rect rkisp1_sd_min_crop = { + .width = RKISP1_ISP_MIN_WIDTH, + .height = RKISP1_ISP_MIN_HEIGHT, + .top = 0, + .left = 0, +}; + +void rkisp1_sd_adjust_crop_rect(struct v4l2_rect *crop, + const struct v4l2_rect *bounds) +{ + v4l2_rect_set_min_size(crop, &rkisp1_sd_min_crop); + v4l2_rect_map_inside(crop, bounds); +} + +void rkisp1_sd_adjust_crop(struct v4l2_rect *crop, + const struct v4l2_mbus_framefmt *bounds) +{ + struct v4l2_rect crop_bounds = { + .left = 0, + .top = 0, + .width = bounds->width, + .height = bounds->height, + }; + + rkisp1_sd_adjust_crop_rect(crop, &crop_bounds); +} diff --git a/drivers/staging/media/rkisp1/rkisp1-common.h b/drivers/staging/media/rkisp1/rkisp1-common.h new file mode 100644 index 000000000000..369a401b098a --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-common.h @@ -0,0 +1,337 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Rockchip ISP1 Driver - Common definitions + * + * Copyright (C) 2019 Collabora, Ltd. + * + * Based on Rockchip ISP1 driver by Rockchip Electronics Co., Ltd. + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#ifndef _RKISP1_COMMON_H +#define _RKISP1_COMMON_H + +#include <linux/clk.h> +#include <linux/mutex.h> +#include <media/media-device.h> +#include <media/media-entity.h> +#include <media/v4l2-ctrls.h> +#include <media/v4l2-device.h> +#include <media/videobuf2-v4l2.h> + +#include "rkisp1-regs.h" +#include "uapi/rkisp1-config.h" + +#define RKISP1_ISP_MAX_WIDTH 4032 +#define RKISP1_ISP_MAX_HEIGHT 3024 +#define RKISP1_ISP_MIN_WIDTH 32 +#define RKISP1_ISP_MIN_HEIGHT 32 + +#define RKISP1_RSZ_MP_SRC_MAX_WIDTH 4416 +#define RKISP1_RSZ_MP_SRC_MAX_HEIGHT 3312 +#define RKISP1_RSZ_SP_SRC_MAX_WIDTH 1920 +#define RKISP1_RSZ_SP_SRC_MAX_HEIGHT 1920 +#define RKISP1_RSZ_SRC_MIN_WIDTH 32 +#define RKISP1_RSZ_SRC_MIN_HEIGHT 16 + +#define RKISP1_DEFAULT_WIDTH 800 +#define RKISP1_DEFAULT_HEIGHT 600 + +#define RKISP1_DRIVER_NAME "rkisp1" +#define RKISP1_BUS_INFO "platform:" RKISP1_DRIVER_NAME + +#define RKISP1_MAX_BUS_CLK 8 + +enum rkisp1_rsz_pad { + RKISP1_RSZ_PAD_SINK, + RKISP1_RSZ_PAD_SRC, +}; + +enum rkisp1_stream_id { + RKISP1_MAINPATH, + RKISP1_SELFPATH, +}; + +enum rkisp1_fmt_pix_type { + RKISP1_FMT_YUV, + RKISP1_FMT_RGB, + RKISP1_FMT_BAYER, + RKISP1_FMT_JPEG, +}; + +enum rkisp1_fmt_raw_pat_type { + RKISP1_RAW_RGGB = 0, + RKISP1_RAW_GRBG, + RKISP1_RAW_GBRG, + RKISP1_RAW_BGGR, +}; + +enum rkisp1_isp_pad { + RKISP1_ISP_PAD_SINK_VIDEO, + RKISP1_ISP_PAD_SINK_PARAMS, + RKISP1_ISP_PAD_SOURCE_VIDEO, + RKISP1_ISP_PAD_SOURCE_STATS, + RKISP1_ISP_PAD_MAX +}; + +/* + * struct rkisp1_sensor_async - Sensor information + * @mbus: media bus configuration + */ +struct rkisp1_sensor_async { + struct v4l2_async_subdev asd; + struct v4l2_mbus_config mbus; + unsigned int lanes; + struct v4l2_subdev *sd; + struct v4l2_ctrl *pixel_rate_ctrl; + struct phy *dphy; +}; + +/* + * struct rkisp1_isp - ISP sub-device + * + * See Cropping regions of ISP in rkisp1.c for details + * @sink_frm: input size, don't have to be equal to sensor size + * @sink_fmt: input format + * @sink_crop: crop for sink pad + * @src_fmt: output format + * @src_crop: output size + * + * @is_dphy_errctrl_disabled : if dphy errctrl is disabled (avoid endless interrupt) + * @frame_sequence: used to synchronize frame_id between video devices. + * @quantization: output quantization + */ +struct rkisp1_isp { + struct v4l2_subdev sd; + struct media_pad pads[RKISP1_ISP_PAD_MAX]; + struct v4l2_subdev_pad_config pad_cfg[RKISP1_ISP_PAD_MAX]; + const struct rkisp1_isp_mbus_info *sink_fmt; + const struct rkisp1_isp_mbus_info *src_fmt; + bool is_dphy_errctrl_disabled; + atomic_t frame_sequence; +}; + +struct rkisp1_vdev_node { + struct vb2_queue buf_queue; + struct mutex vlock; /* ioctl serialization mutex */ + struct video_device vdev; + struct media_pad pad; +}; + +struct rkisp1_buffer { + struct vb2_v4l2_buffer vb; + struct list_head queue; + union { + u32 buff_addr[VIDEO_MAX_PLANES]; + void *vaddr[VIDEO_MAX_PLANES]; + }; +}; + +struct rkisp1_dummy_buffer { + void *vaddr; + dma_addr_t dma_addr; + u32 size; +}; + +struct rkisp1_device; + +/* + * struct rkisp1_capture - ISP capture video device + * + * @pix.fmt: buffer format + * @pix.info: pixel information + * @pix.cfg: pixel configuration + * + * @buf.lock: lock to protect buf_queue + * @buf.queue: queued buffer list + * @buf.dummy: dummy space to store dropped data + * + * rkisp1 use shadowsock registers, so it need two buffer at a time + * @buf.curr: the buffer used for current frame + * @buf.next: the buffer used for next frame + */ +struct rkisp1_capture { + struct rkisp1_vdev_node vnode; + struct rkisp1_device *rkisp1; + enum rkisp1_stream_id id; + struct rkisp1_capture_ops *ops; + const struct rkisp1_capture_config *config; + bool is_streaming; + bool is_stopping; + wait_queue_head_t done; + unsigned int sp_y_stride; + struct { + /* protects queue, curr and next */ + spinlock_t lock; + struct list_head queue; + struct rkisp1_dummy_buffer dummy; + struct rkisp1_buffer *curr; + struct rkisp1_buffer *next; + } buf; + struct { + const struct rkisp1_capture_fmt_cfg *cfg; + const struct v4l2_format_info *info; + struct v4l2_pix_format_mplane fmt; + } pix; +}; + +/* + * struct rkisp1_stats - ISP Statistics device + * + * @irq_lock: buffer queue lock + * @stat: stats buffer list + * @readout_wq: workqueue for statistics information read + */ +struct rkisp1_stats { + struct rkisp1_vdev_node vnode; + struct rkisp1_device *rkisp1; + + spinlock_t irq_lock; + struct list_head stat; + struct v4l2_format vdev_fmt; + bool is_streaming; + + struct workqueue_struct *readout_wq; + struct mutex wq_lock; +}; + +/* + * struct rkisp1_params - ISP input parameters device + * + * @cur_params: Current ISP parameters + * @is_first_params: the first params should take effect immediately + */ +struct rkisp1_params { + struct rkisp1_vdev_node vnode; + struct rkisp1_device *rkisp1; + + spinlock_t config_lock; + struct list_head params; + struct rkisp1_params_cfg cur_params; + struct v4l2_format vdev_fmt; + bool is_streaming; + bool is_first_params; + + enum v4l2_quantization quantization; + enum rkisp1_fmt_raw_pat_type raw_type; +}; + +struct rkisp1_resizer { + struct v4l2_subdev sd; + enum rkisp1_stream_id id; + struct rkisp1_device *rkisp1; + struct media_pad pads[RKISP1_ISP_PAD_MAX]; + struct v4l2_subdev_pad_config pad_cfg[RKISP1_ISP_PAD_MAX]; + const struct rkisp1_rsz_config *config; + enum rkisp1_fmt_pix_type fmt_type; +}; + +struct rkisp1_debug { + struct dentry *debugfs_dir; + unsigned long data_loss; + unsigned long pic_size_error; + unsigned long mipi_error; + unsigned long stats_error; + unsigned long stop_timeout[2]; + unsigned long frame_drop[2]; +}; + +/* + * struct rkisp1_device - ISP platform device + * @base_addr: base register address + * @active_sensor: sensor in-use, set when streaming on + * @isp: ISP sub-device + * @rkisp1_capture: capture video device + * @stats: ISP statistics output device + * @params: ISP input parameters device + */ +struct rkisp1_device { + void __iomem *base_addr; + int irq; + struct device *dev; + unsigned int clk_size; + struct clk_bulk_data clks[RKISP1_MAX_BUS_CLK]; + struct v4l2_device v4l2_dev; + struct v4l2_ctrl_handler ctrl_handler; + struct media_device media_dev; + struct v4l2_async_notifier notifier; + struct rkisp1_sensor_async *active_sensor; + struct rkisp1_isp isp; + struct rkisp1_resizer resizer_devs[2]; + struct rkisp1_capture capture_devs[2]; + struct rkisp1_stats stats; + struct rkisp1_params params; + struct media_pipeline pipe; + struct vb2_alloc_ctx *alloc_ctx; + struct rkisp1_debug debug; +}; + +/* + * struct rkisp1_isp_mbus_info - ISP pad format info + * + * Translate mbus_code to hardware format values + * + * @bus_width: used for parallel + */ +struct rkisp1_isp_mbus_info { + u32 mbus_code; + enum rkisp1_fmt_pix_type fmt_type; + u32 mipi_dt; + u32 yuv_seq; + u8 bus_width; + enum rkisp1_fmt_raw_pat_type bayer_pat; + unsigned int direction; +}; + +static inline void +rkisp1_write(struct rkisp1_device *rkisp1, u32 val, unsigned int addr) +{ + writel(val, rkisp1->base_addr + addr); +} + +static inline u32 rkisp1_read(struct rkisp1_device *rkisp1, unsigned int addr) +{ + return readl(rkisp1->base_addr + addr); +} + +void rkisp1_sd_adjust_crop_rect(struct v4l2_rect *crop, + const struct v4l2_rect *bounds); + +void rkisp1_sd_adjust_crop(struct v4l2_rect *crop, + const struct v4l2_mbus_framefmt *bounds); + +int rkisp1_isp_register(struct rkisp1_device *rkisp1, + struct v4l2_device *v4l2_dev); +void rkisp1_isp_unregister(struct rkisp1_device *rkisp1); + +const struct rkisp1_isp_mbus_info *rkisp1_isp_mbus_info_get(u32 mbus_code); + +void rkisp1_isp_isr(struct rkisp1_device *rkisp1); +void rkisp1_mipi_isr(struct rkisp1_device *rkisp1); +void rkisp1_capture_isr(struct rkisp1_device *rkisp1); +void rkisp1_stats_isr(struct rkisp1_stats *stats, u32 isp_ris); +void rkisp1_params_isr(struct rkisp1_device *rkisp1, u32 isp_mis); + +int rkisp1_capture_devs_register(struct rkisp1_device *rkisp1); +void rkisp1_capture_devs_unregister(struct rkisp1_device *rkisp1); + +int rkisp1_resizer_devs_register(struct rkisp1_device *rkisp1); +void rkisp1_resizer_devs_unregister(struct rkisp1_device *rkisp1); + +int rkisp1_stats_register(struct rkisp1_stats *stats, + struct v4l2_device *v4l2_dev, + struct rkisp1_device *rkisp1); +void rkisp1_stats_unregister(struct rkisp1_stats *stats); + +void rkisp1_params_configure(struct rkisp1_params *params, + enum rkisp1_fmt_raw_pat_type bayer_pat, + enum v4l2_quantization quantization); +void rkisp1_params_disable(struct rkisp1_params *params); +int rkisp1_params_register(struct rkisp1_params *params, + struct v4l2_device *v4l2_dev, + struct rkisp1_device *rkisp1); +void rkisp1_params_unregister(struct rkisp1_params *params); + +void rkisp1_params_isr_handler(struct rkisp1_device *rkisp1, u32 isp_mis); + +#endif /* _RKISP1_COMMON_H */ diff --git a/drivers/staging/media/rkisp1/rkisp1-dev.c b/drivers/staging/media/rkisp1/rkisp1-dev.c new file mode 100644 index 000000000000..558126e66465 --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-dev.c @@ -0,0 +1,574 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip ISP1 Driver - Base driver + * + * Copyright (C) 2019 Collabora, Ltd. + * + * Based on Rockchip ISP1 driver by Rockchip Electronics Co., Ltd. + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#include <linux/clk.h> +#include <linux/debugfs.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_graph.h> +#include <linux/of_platform.h> +#include <linux/pinctrl/consumer.h> +#include <linux/phy/phy.h> +#include <linux/phy/phy-mipi-dphy.h> +#include <media/v4l2-fwnode.h> + +#include "rkisp1-common.h" + +/* + * ISP Details + * ----------- + * + * ISP Comprises with: + * MIPI serial camera interface + * Image Signal Processing + * Many Image Enhancement Blocks + * Crop + * Resizer + * RBG display ready image + * Image Rotation + * + * ISP Block Diagram + * ----------------- + * rkisp1-resizer.c rkisp1-capture.c + * |====================| |=======================| + * rkisp1-isp.c Main Picture Path + * |==========================| |===============================================| + * +-----------+ +--+--+--+--+ +--------+ +--------+ +-----------+ + * | | | | | | | | | | | | | + * +--------+ |\ | | | | | | | -->| Crop |->| RSZ |------------->| | + * | MIPI |--->| \ | | | | | | | | | | | | | | + * +--------+ | | | | |IE|IE|IE|IE| | +--------+ +--------+ | Memory | + * |MUX|--->| ISP |->|0 |1 |2 |3 |---+ | Interface | + * +--------+ | | | | | | | | | | +--------+ +--------+ +--------+ | | + * |Parallel|--->| / | | | | | | | | | | | | | | | | + * +--------+ |/ | | | | | | | -->| Crop |->| RSZ |->| RGB |->| | + * | | | | | | | | | | | | Rotate | | | + * +-----------+ +--+--+--+--+ +--------+ +--------+ +--------+ +-----------+ + * ^ + * +--------+ | |===============================================| + * | DMA |------------------------------------+ Self Picture Path + * +--------+ + * + * rkisp1-stats.c rkisp1-params.c + * |===============| |===============| + * +---------------+ +---------------+ + * | | | | + * | ISP | | ISP | + * | | | | + * +---------------+ +---------------+ + * + * + * Media Topology + * -------------- + * +----------+ +----------+ + * | Sensor 2 | | Sensor X | + * ------------ ... ------------ + * | 0 | | 0 | + * +----------+ +----------+ +-----------+ + * \ | | params | + * \ | | (output) | + * +----------+ \ | +-----------+ + * | Sensor 1 | v v | + * ------------ +------+------+ | + * | 0 |----->| 0 | 1 |<---------+ + * +----------+ |------+------| + * | ISP | + * |------+------| + * +-------------| 2 | 3 |----------+ + * | +------+------+ | + * | | | + * v v v + * +- ---------+ +-----------+ +-----------+ + * | 0 | | 0 | | stats | + * ------------- ------------- | (capture) | + * | Resizer | | Resizer | +-----------+ + * ------------| ------------| + * | 1 | | 1 | + * +-----------+ +-----------+ + * | | + * v v + * +-----------+ +-----------+ + * | selfpath | | mainpath | + * | (capture) | | (capture) | + * +-----------+ +-----------+ + */ + +struct rkisp1_match_data { + const char * const *clks; + unsigned int size; +}; + +/* ---------------------------------------------------------------------------- + * Sensor DT bindings + */ + +static int rkisp1_create_links(struct rkisp1_device *rkisp1) +{ + struct media_entity *source, *sink; + unsigned int flags, source_pad; + struct v4l2_subdev *sd; + unsigned int i; + int ret; + + /* sensor links */ + flags = MEDIA_LNK_FL_ENABLED; + list_for_each_entry(sd, &rkisp1->v4l2_dev.subdevs, list) { + if (sd == &rkisp1->isp.sd || + sd == &rkisp1->resizer_devs[RKISP1_MAINPATH].sd || + sd == &rkisp1->resizer_devs[RKISP1_SELFPATH].sd) + continue; + + ret = media_entity_get_fwnode_pad(&sd->entity, sd->fwnode, + MEDIA_PAD_FL_SOURCE); + if (ret) { + dev_err(sd->dev, "failed to find src pad for %s\n", + sd->name); + return ret; + } + source_pad = ret; + + ret = media_create_pad_link(&sd->entity, source_pad, + &rkisp1->isp.sd.entity, + RKISP1_ISP_PAD_SINK_VIDEO, + flags); + if (ret) + return ret; + + flags = 0; + } + + flags = MEDIA_LNK_FL_ENABLED; + + /* create ISP->RSZ->CAP links */ + for (i = 0; i < 2; i++) { + source = &rkisp1->isp.sd.entity; + sink = &rkisp1->resizer_devs[i].sd.entity; + ret = media_create_pad_link(source, RKISP1_ISP_PAD_SOURCE_VIDEO, + sink, RKISP1_RSZ_PAD_SINK, flags); + if (ret) + return ret; + + source = sink; + sink = &rkisp1->capture_devs[i].vnode.vdev.entity; + ret = media_create_pad_link(source, RKISP1_RSZ_PAD_SRC, + sink, 0, flags); + if (ret) + return ret; + } + + /* params links */ + source = &rkisp1->params.vnode.vdev.entity; + sink = &rkisp1->isp.sd.entity; + ret = media_create_pad_link(source, 0, sink, + RKISP1_ISP_PAD_SINK_PARAMS, flags); + if (ret) + return ret; + + /* 3A stats links */ + source = &rkisp1->isp.sd.entity; + sink = &rkisp1->stats.vnode.vdev.entity; + return media_create_pad_link(source, RKISP1_ISP_PAD_SOURCE_STATS, + sink, 0, flags); +} + +static int rkisp1_subdev_notifier_bound(struct v4l2_async_notifier *notifier, + struct v4l2_subdev *sd, + struct v4l2_async_subdev *asd) +{ + struct rkisp1_device *rkisp1 = + container_of(notifier, struct rkisp1_device, notifier); + struct rkisp1_sensor_async *s_asd = + container_of(asd, struct rkisp1_sensor_async, asd); + + s_asd->pixel_rate_ctrl = v4l2_ctrl_find(sd->ctrl_handler, + V4L2_CID_PIXEL_RATE); + s_asd->sd = sd; + s_asd->dphy = devm_phy_get(rkisp1->dev, "dphy"); + if (IS_ERR(s_asd->dphy)) { + if (PTR_ERR(s_asd->dphy) != -EPROBE_DEFER) + dev_err(rkisp1->dev, "Couldn't get the MIPI D-PHY\n"); + return PTR_ERR(s_asd->dphy); + } + + phy_init(s_asd->dphy); + + return 0; +} + +static void rkisp1_subdev_notifier_unbind(struct v4l2_async_notifier *notifier, + struct v4l2_subdev *sd, + struct v4l2_async_subdev *asd) +{ + struct rkisp1_sensor_async *s_asd = + container_of(asd, struct rkisp1_sensor_async, asd); + + phy_exit(s_asd->dphy); +} + +static int rkisp1_subdev_notifier_complete(struct v4l2_async_notifier *notifier) +{ + struct rkisp1_device *rkisp1 = + container_of(notifier, struct rkisp1_device, notifier); + int ret; + + mutex_lock(&rkisp1->media_dev.graph_mutex); + ret = rkisp1_create_links(rkisp1); + if (ret) + goto unlock; + ret = v4l2_device_register_subdev_nodes(&rkisp1->v4l2_dev); + if (ret) + goto unlock; + + dev_dbg(rkisp1->dev, "Async subdev notifier completed\n"); + +unlock: + mutex_unlock(&rkisp1->media_dev.graph_mutex); + return ret; +} + +static int rkisp1_fwnode_parse(struct device *dev, + struct v4l2_fwnode_endpoint *vep, + struct v4l2_async_subdev *asd) +{ + struct rkisp1_sensor_async *s_asd = + container_of(asd, struct rkisp1_sensor_async, asd); + + if (vep->bus_type != V4L2_MBUS_CSI2_DPHY) { + dev_err(dev, "Only CSI2 bus type is currently supported\n"); + return -EINVAL; + } + + if (vep->base.port != 0) { + dev_err(dev, "The ISP has only port 0\n"); + return -EINVAL; + } + + s_asd->mbus.type = vep->bus_type; + s_asd->mbus.flags = vep->bus.mipi_csi2.flags; + s_asd->lanes = vep->bus.mipi_csi2.num_data_lanes; + + switch (vep->bus.mipi_csi2.num_data_lanes) { + case 1: + s_asd->mbus.flags |= V4L2_MBUS_CSI2_1_LANE; + break; + case 2: + s_asd->mbus.flags |= V4L2_MBUS_CSI2_2_LANE; + break; + case 3: + s_asd->mbus.flags |= V4L2_MBUS_CSI2_3_LANE; + break; + case 4: + s_asd->mbus.flags |= V4L2_MBUS_CSI2_4_LANE; + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct v4l2_async_notifier_operations rkisp1_subdev_notifier_ops = { + .bound = rkisp1_subdev_notifier_bound, + .unbind = rkisp1_subdev_notifier_unbind, + .complete = rkisp1_subdev_notifier_complete, +}; + +static int rkisp1_subdev_notifier(struct rkisp1_device *rkisp1) +{ + struct v4l2_async_notifier *ntf = &rkisp1->notifier; + struct device *dev = rkisp1->dev; + int ret; + + v4l2_async_notifier_init(ntf); + + ret = v4l2_async_notifier_parse_fwnode_endpoints_by_port(dev, ntf, + sizeof(struct rkisp1_sensor_async), + 0, rkisp1_fwnode_parse); + if (ret) + return ret; + + if (list_empty(&ntf->asd_list)) + return -ENODEV; + + ntf->ops = &rkisp1_subdev_notifier_ops; + + return v4l2_async_notifier_register(&rkisp1->v4l2_dev, ntf); +} + +/* ---------------------------------------------------------------------------- + * Power + */ + +static int __maybe_unused rkisp1_runtime_suspend(struct device *dev) +{ + struct rkisp1_device *rkisp1 = dev_get_drvdata(dev); + + clk_bulk_disable_unprepare(rkisp1->clk_size, rkisp1->clks); + return pinctrl_pm_select_sleep_state(dev); +} + +static int __maybe_unused rkisp1_runtime_resume(struct device *dev) +{ + struct rkisp1_device *rkisp1 = dev_get_drvdata(dev); + int ret; + + ret = pinctrl_pm_select_default_state(dev); + if (ret) + return ret; + ret = clk_bulk_prepare_enable(rkisp1->clk_size, rkisp1->clks); + if (ret) + return ret; + + return 0; +} + +static const struct dev_pm_ops rkisp1_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(rkisp1_runtime_suspend, rkisp1_runtime_resume, NULL) +}; + +/* ---------------------------------------------------------------------------- + * Core + */ + +static int rkisp1_entities_register(struct rkisp1_device *rkisp1) +{ + int ret; + + ret = rkisp1_isp_register(rkisp1, &rkisp1->v4l2_dev); + if (ret) + return ret; + + ret = rkisp1_resizer_devs_register(rkisp1); + if (ret) + goto err_unreg_isp_subdev; + + ret = rkisp1_capture_devs_register(rkisp1); + if (ret) + goto err_unreg_resizer_devs; + + ret = rkisp1_stats_register(&rkisp1->stats, &rkisp1->v4l2_dev, rkisp1); + if (ret) + goto err_unreg_capture_devs; + + ret = rkisp1_params_register(&rkisp1->params, + &rkisp1->v4l2_dev, rkisp1); + if (ret) + goto err_unreg_stats; + + ret = rkisp1_subdev_notifier(rkisp1); + if (ret) { + dev_err(rkisp1->dev, + "Failed to register subdev notifier(%d)\n", ret); + goto err_unreg_params; + } + + return 0; +err_unreg_params: + rkisp1_params_unregister(&rkisp1->params); +err_unreg_stats: + rkisp1_stats_unregister(&rkisp1->stats); +err_unreg_capture_devs: + rkisp1_capture_devs_unregister(rkisp1); +err_unreg_resizer_devs: + rkisp1_resizer_devs_unregister(rkisp1); +err_unreg_isp_subdev: + rkisp1_isp_unregister(rkisp1); + return ret; +} + +static irqreturn_t rkisp1_isr(int irq, void *ctx) +{ + struct device *dev = ctx; + struct rkisp1_device *rkisp1 = dev_get_drvdata(dev); + + /* + * Call rkisp1_capture_isr() first to handle the frame that + * potentially completed using the current frame_sequence number before + * it is potentially incremented by rkisp1_isp_isr() in the vertical + * sync. + */ + rkisp1_capture_isr(rkisp1); + rkisp1_isp_isr(rkisp1); + rkisp1_mipi_isr(rkisp1); + + return IRQ_HANDLED; +} + +static const char * const rk3399_isp_clks[] = { + "clk_isp", + "aclk_isp", + "hclk_isp", + "aclk_isp_wrap", + "hclk_isp_wrap", +}; + +static const struct rkisp1_match_data rk3399_isp_clk_data = { + .clks = rk3399_isp_clks, + .size = ARRAY_SIZE(rk3399_isp_clks), +}; + +static const struct of_device_id rkisp1_of_match[] = { + { + .compatible = "rockchip,rk3399-cif-isp", + .data = &rk3399_isp_clk_data, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, rkisp1_of_match); + +static void rkisp1_debug_init(struct rkisp1_device *rkisp1) +{ + struct rkisp1_debug *debug = &rkisp1->debug; + + debug->debugfs_dir = debugfs_create_dir(RKISP1_DRIVER_NAME, NULL); + if (!debug->debugfs_dir) { + dev_dbg(rkisp1->dev, "failed to create debugfs directory\n"); + return; + } + debugfs_create_ulong("data_loss", 0444, debug->debugfs_dir, + &debug->data_loss); + debugfs_create_ulong("pic_size_error", 0444, debug->debugfs_dir, + &debug->pic_size_error); + debugfs_create_ulong("mipi_error", 0444, debug->debugfs_dir, + &debug->mipi_error); + debugfs_create_ulong("stats_error", 0444, debug->debugfs_dir, + &debug->stats_error); + debugfs_create_ulong("mp_stop_timeout", 0444, debug->debugfs_dir, + &debug->stop_timeout[RKISP1_MAINPATH]); + debugfs_create_ulong("sp_stop_timeout", 0444, debug->debugfs_dir, + &debug->stop_timeout[RKISP1_SELFPATH]); + debugfs_create_ulong("mp_frame_drop", 0444, debug->debugfs_dir, + &debug->frame_drop[RKISP1_MAINPATH]); + debugfs_create_ulong("sp_frame_drop", 0444, debug->debugfs_dir, + &debug->frame_drop[RKISP1_SELFPATH]); +} + +static int rkisp1_probe(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + const struct rkisp1_match_data *clk_data; + const struct of_device_id *match; + struct device *dev = &pdev->dev; + struct rkisp1_device *rkisp1; + struct v4l2_device *v4l2_dev; + unsigned int i; + int ret, irq; + + match = of_match_node(rkisp1_of_match, node); + rkisp1 = devm_kzalloc(dev, sizeof(*rkisp1), GFP_KERNEL); + if (!rkisp1) + return -ENOMEM; + + dev_set_drvdata(dev, rkisp1); + rkisp1->dev = dev; + + rkisp1->base_addr = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(rkisp1->base_addr)) + return PTR_ERR(rkisp1->base_addr); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, rkisp1_isr, IRQF_SHARED, + dev_driver_string(dev), dev); + if (ret) { + dev_err(dev, "request irq failed: %d\n", ret); + return ret; + } + + rkisp1->irq = irq; + clk_data = match->data; + + for (i = 0; i < clk_data->size; i++) + rkisp1->clks[i].id = clk_data->clks[i]; + ret = devm_clk_bulk_get(dev, clk_data->size, rkisp1->clks); + if (ret) + return ret; + rkisp1->clk_size = clk_data->size; + + pm_runtime_enable(&pdev->dev); + + strscpy(rkisp1->media_dev.model, RKISP1_DRIVER_NAME, + sizeof(rkisp1->media_dev.model)); + rkisp1->media_dev.dev = &pdev->dev; + strscpy(rkisp1->media_dev.bus_info, + "platform: " RKISP1_DRIVER_NAME, + sizeof(rkisp1->media_dev.bus_info)); + media_device_init(&rkisp1->media_dev); + + v4l2_dev = &rkisp1->v4l2_dev; + v4l2_dev->mdev = &rkisp1->media_dev; + strscpy(v4l2_dev->name, RKISP1_DRIVER_NAME, sizeof(v4l2_dev->name)); + + ret = v4l2_device_register(rkisp1->dev, &rkisp1->v4l2_dev); + if (ret) + return ret; + + ret = media_device_register(&rkisp1->media_dev); + if (ret) { + dev_err(dev, "Failed to register media device: %d\n", ret); + goto err_unreg_v4l2_dev; + } + + ret = rkisp1_entities_register(rkisp1); + if (ret) + goto err_unreg_media_dev; + + rkisp1_debug_init(rkisp1); + + return 0; + +err_unreg_media_dev: + media_device_unregister(&rkisp1->media_dev); +err_unreg_v4l2_dev: + v4l2_device_unregister(&rkisp1->v4l2_dev); + pm_runtime_disable(&pdev->dev); + return ret; +} + +static int rkisp1_remove(struct platform_device *pdev) +{ + struct rkisp1_device *rkisp1 = platform_get_drvdata(pdev); + + v4l2_async_notifier_unregister(&rkisp1->notifier); + v4l2_async_notifier_cleanup(&rkisp1->notifier); + + rkisp1_params_unregister(&rkisp1->params); + rkisp1_stats_unregister(&rkisp1->stats); + rkisp1_capture_devs_unregister(rkisp1); + rkisp1_resizer_devs_unregister(rkisp1); + rkisp1_isp_unregister(rkisp1); + + media_device_unregister(&rkisp1->media_dev); + v4l2_device_unregister(&rkisp1->v4l2_dev); + + pm_runtime_disable(&pdev->dev); + + debugfs_remove_recursive(rkisp1->debug.debugfs_dir); + return 0; +} + +static struct platform_driver rkisp1_drv = { + .driver = { + .name = RKISP1_DRIVER_NAME, + .of_match_table = of_match_ptr(rkisp1_of_match), + .pm = &rkisp1_pm_ops, + }, + .probe = rkisp1_probe, + .remove = rkisp1_remove, +}; + +module_platform_driver(rkisp1_drv); +MODULE_DESCRIPTION("Rockchip ISP1 platform driver"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/staging/media/rkisp1/rkisp1-isp.c b/drivers/staging/media/rkisp1/rkisp1-isp.c new file mode 100644 index 000000000000..328c7ea60971 --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-isp.c @@ -0,0 +1,1164 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip ISP1 Driver - ISP Subdevice + * + * Copyright (C) 2019 Collabora, Ltd. + * + * Based on Rockchip ISP1 driver by Rockchip Electronics Co., Ltd. + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#include <linux/iopoll.h> +#include <linux/phy/phy.h> +#include <linux/phy/phy-mipi-dphy.h> +#include <linux/pm_runtime.h> +#include <linux/videodev2.h> +#include <linux/vmalloc.h> +#include <media/v4l2-event.h> + +#include "rkisp1-common.h" + +#define RKISP1_DEF_SINK_PAD_FMT MEDIA_BUS_FMT_SRGGB10_1X10 +#define RKISP1_DEF_SRC_PAD_FMT MEDIA_BUS_FMT_YUYV8_2X8 + +#define RKISP1_ISP_DEV_NAME RKISP1_DRIVER_NAME "_isp" + +#define RKISP1_DIR_SRC BIT(0) +#define RKISP1_DIR_SINK BIT(1) +#define RKISP1_DIR_SINK_SRC (RKISP1_DIR_SINK | RKISP1_DIR_SRC) + +/* + * NOTE: MIPI controller and input MUX are also configured in this file, + * because ISP Subdev is not only describe ISP submodule(input size,format, + * output size, format), but also a virtual route device. + */ + +/* + * There are many variables named with format/frame in below code, + * please see here for their meaning. + * Cropping in the sink pad defines the image region from the sensor. + * Cropping in the source pad defines the region for the Image Stabilizer (IS) + * + * Cropping regions of ISP + * + * +---------------------------------------------------------+ + * | Sensor image | + * | +---------------------------------------------------+ | + * | | CIF_ISP_ACQ (for black level) | | + * | | sink pad format | | + * | | +--------------------------------------------+ | | + * | | | CIF_ISP_OUT | | | + * | | | sink pad crop | | | + * | | | +---------------------------------+ | | | + * | | | | CIF_ISP_IS | | | | + * | | | | source pad crop and format | | | | + * | | | +---------------------------------+ | | | + * | | +--------------------------------------------+ | | + * | +---------------------------------------------------+ | + * +---------------------------------------------------------+ + */ + +static const struct rkisp1_isp_mbus_info rkisp1_isp_formats[] = { + { + .mbus_code = MEDIA_BUS_FMT_YUYV8_2X8, + .fmt_type = RKISP1_FMT_YUV, + .direction = RKISP1_DIR_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW10, + .bayer_pat = RKISP1_RAW_RGGB, + .bus_width = 10, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW10, + .bayer_pat = RKISP1_RAW_BGGR, + .bus_width = 10, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW10, + .bayer_pat = RKISP1_RAW_GBRG, + .bus_width = 10, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW10, + .bayer_pat = RKISP1_RAW_GRBG, + .bus_width = 10, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW12, + .bayer_pat = RKISP1_RAW_RGGB, + .bus_width = 12, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW12, + .bayer_pat = RKISP1_RAW_BGGR, + .bus_width = 12, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW12, + .bayer_pat = RKISP1_RAW_GBRG, + .bus_width = 12, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW12, + .bayer_pat = RKISP1_RAW_GRBG, + .bus_width = 12, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW8, + .bayer_pat = RKISP1_RAW_RGGB, + .bus_width = 8, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW8, + .bayer_pat = RKISP1_RAW_BGGR, + .bus_width = 8, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW8, + .bayer_pat = RKISP1_RAW_GBRG, + .bus_width = 8, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8, + .fmt_type = RKISP1_FMT_BAYER, + .mipi_dt = RKISP1_CIF_CSI2_DT_RAW8, + .bayer_pat = RKISP1_RAW_GRBG, + .bus_width = 8, + .direction = RKISP1_DIR_SINK_SRC, + }, { + .mbus_code = MEDIA_BUS_FMT_YUYV8_1X16, + .fmt_type = RKISP1_FMT_YUV, + .mipi_dt = RKISP1_CIF_CSI2_DT_YUV422_8b, + .yuv_seq = RKISP1_CIF_ISP_ACQ_PROP_YCBYCR, + .bus_width = 16, + .direction = RKISP1_DIR_SINK, + }, { + .mbus_code = MEDIA_BUS_FMT_YVYU8_1X16, + .fmt_type = RKISP1_FMT_YUV, + .mipi_dt = RKISP1_CIF_CSI2_DT_YUV422_8b, + .yuv_seq = RKISP1_CIF_ISP_ACQ_PROP_YCRYCB, + .bus_width = 16, + .direction = RKISP1_DIR_SINK, + }, { + .mbus_code = MEDIA_BUS_FMT_UYVY8_1X16, + .fmt_type = RKISP1_FMT_YUV, + .mipi_dt = RKISP1_CIF_CSI2_DT_YUV422_8b, + .yuv_seq = RKISP1_CIF_ISP_ACQ_PROP_CBYCRY, + .bus_width = 16, + .direction = RKISP1_DIR_SINK, + }, { + .mbus_code = MEDIA_BUS_FMT_VYUY8_1X16, + .fmt_type = RKISP1_FMT_YUV, + .mipi_dt = RKISP1_CIF_CSI2_DT_YUV422_8b, + .yuv_seq = RKISP1_CIF_ISP_ACQ_PROP_CRYCBY, + .bus_width = 16, + .direction = RKISP1_DIR_SINK, + }, +}; + +/* ---------------------------------------------------------------------------- + * Helpers + */ + +const struct rkisp1_isp_mbus_info *rkisp1_isp_mbus_info_get(u32 mbus_code) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rkisp1_isp_formats); i++) { + const struct rkisp1_isp_mbus_info *fmt = &rkisp1_isp_formats[i]; + + if (fmt->mbus_code == mbus_code) + return fmt; + } + + return NULL; +} + +static struct v4l2_subdev *rkisp1_get_remote_sensor(struct v4l2_subdev *sd) +{ + struct media_pad *local, *remote; + struct media_entity *sensor_me; + + local = &sd->entity.pads[RKISP1_ISP_PAD_SINK_VIDEO]; + remote = media_entity_remote_pad(local); + if (!remote) { + dev_warn(sd->dev, "No link between isp and sensor\n"); + return NULL; + } + + sensor_me = remote->entity; + return media_entity_to_v4l2_subdev(sensor_me); +} + +static struct v4l2_mbus_framefmt * +rkisp1_isp_get_pad_fmt(struct rkisp1_isp *isp, + struct v4l2_subdev_pad_config *cfg, + unsigned int pad, u32 which) +{ + if (which == V4L2_SUBDEV_FORMAT_TRY) + return v4l2_subdev_get_try_format(&isp->sd, cfg, pad); + else + return v4l2_subdev_get_try_format(&isp->sd, isp->pad_cfg, pad); +} + +static struct v4l2_rect * +rkisp1_isp_get_pad_crop(struct rkisp1_isp *isp, + struct v4l2_subdev_pad_config *cfg, + unsigned int pad, u32 which) +{ + if (which == V4L2_SUBDEV_FORMAT_TRY) + return v4l2_subdev_get_try_crop(&isp->sd, cfg, pad); + else + return v4l2_subdev_get_try_crop(&isp->sd, isp->pad_cfg, pad); +} + +/* ---------------------------------------------------------------------------- + * Camera Interface registers configurations + */ + +/* + * Image Stabilization. + * This should only be called when configuring CIF + * or at the frame end interrupt + */ +static void rkisp1_config_ism(struct rkisp1_device *rkisp1) +{ + struct v4l2_rect *src_crop = + rkisp1_isp_get_pad_crop(&rkisp1->isp, NULL, + RKISP1_ISP_PAD_SOURCE_VIDEO, + V4L2_SUBDEV_FORMAT_ACTIVE); + u32 val; + + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_IS_RECENTER); + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_IS_MAX_DX); + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_IS_MAX_DY); + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_IS_DISPLACE); + rkisp1_write(rkisp1, src_crop->left, RKISP1_CIF_ISP_IS_H_OFFS); + rkisp1_write(rkisp1, src_crop->top, RKISP1_CIF_ISP_IS_V_OFFS); + rkisp1_write(rkisp1, src_crop->width, RKISP1_CIF_ISP_IS_H_SIZE); + rkisp1_write(rkisp1, src_crop->height, RKISP1_CIF_ISP_IS_V_SIZE); + + /* IS(Image Stabilization) is always on, working as output crop */ + rkisp1_write(rkisp1, 1, RKISP1_CIF_ISP_IS_CTRL); + val = rkisp1_read(rkisp1, RKISP1_CIF_ISP_CTRL); + val |= RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD; + rkisp1_write(rkisp1, val, RKISP1_CIF_ISP_CTRL); +} + +/* + * configure ISP blocks with input format, size...... + */ +static int rkisp1_config_isp(struct rkisp1_device *rkisp1) +{ + u32 isp_ctrl = 0, irq_mask = 0, acq_mult = 0, signal = 0; + const struct rkisp1_isp_mbus_info *src_fmt, *sink_fmt; + struct rkisp1_sensor_async *sensor; + struct v4l2_mbus_framefmt *sink_frm; + struct v4l2_rect *sink_crop; + + sensor = rkisp1->active_sensor; + sink_fmt = rkisp1->isp.sink_fmt; + src_fmt = rkisp1->isp.src_fmt; + sink_frm = rkisp1_isp_get_pad_fmt(&rkisp1->isp, NULL, + RKISP1_ISP_PAD_SINK_VIDEO, + V4L2_SUBDEV_FORMAT_ACTIVE); + sink_crop = rkisp1_isp_get_pad_crop(&rkisp1->isp, NULL, + RKISP1_ISP_PAD_SINK_VIDEO, + V4L2_SUBDEV_FORMAT_ACTIVE); + + if (sink_fmt->fmt_type == RKISP1_FMT_BAYER) { + acq_mult = 1; + if (src_fmt->fmt_type == RKISP1_FMT_BAYER) { + if (sensor->mbus.type == V4L2_MBUS_BT656) + isp_ctrl = RKISP1_CIF_ISP_CTRL_ISP_MODE_RAW_PICT_ITU656; + else + isp_ctrl = RKISP1_CIF_ISP_CTRL_ISP_MODE_RAW_PICT; + } else { + rkisp1_write(rkisp1, RKISP1_CIF_ISP_DEMOSAIC_TH(0xc), + RKISP1_CIF_ISP_DEMOSAIC); + + if (sensor->mbus.type == V4L2_MBUS_BT656) + isp_ctrl = RKISP1_CIF_ISP_CTRL_ISP_MODE_BAYER_ITU656; + else + isp_ctrl = RKISP1_CIF_ISP_CTRL_ISP_MODE_BAYER_ITU601; + } + } else if (sink_fmt->fmt_type == RKISP1_FMT_YUV) { + acq_mult = 2; + if (sensor->mbus.type == V4L2_MBUS_CSI2_DPHY) { + isp_ctrl = RKISP1_CIF_ISP_CTRL_ISP_MODE_ITU601; + } else { + if (sensor->mbus.type == V4L2_MBUS_BT656) + isp_ctrl = RKISP1_CIF_ISP_CTRL_ISP_MODE_ITU656; + else + isp_ctrl = RKISP1_CIF_ISP_CTRL_ISP_MODE_ITU601; + } + + irq_mask |= RKISP1_CIF_ISP_DATA_LOSS; + } + + /* Set up input acquisition properties */ + if (sensor->mbus.type == V4L2_MBUS_BT656 || + sensor->mbus.type == V4L2_MBUS_PARALLEL) { + if (sensor->mbus.flags & V4L2_MBUS_PCLK_SAMPLE_RISING) + signal = RKISP1_CIF_ISP_ACQ_PROP_POS_EDGE; + } + + if (sensor->mbus.type == V4L2_MBUS_PARALLEL) { + if (sensor->mbus.flags & V4L2_MBUS_VSYNC_ACTIVE_LOW) + signal |= RKISP1_CIF_ISP_ACQ_PROP_VSYNC_LOW; + + if (sensor->mbus.flags & V4L2_MBUS_HSYNC_ACTIVE_LOW) + signal |= RKISP1_CIF_ISP_ACQ_PROP_HSYNC_LOW; + } + + rkisp1_write(rkisp1, isp_ctrl, RKISP1_CIF_ISP_CTRL); + rkisp1_write(rkisp1, signal | sink_fmt->yuv_seq | + RKISP1_CIF_ISP_ACQ_PROP_BAYER_PAT(sink_fmt->bayer_pat) | + RKISP1_CIF_ISP_ACQ_PROP_FIELD_SEL_ALL, + RKISP1_CIF_ISP_ACQ_PROP); + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_ACQ_NR_FRAMES); + + /* Acquisition Size */ + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_ACQ_H_OFFS); + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_ACQ_V_OFFS); + rkisp1_write(rkisp1, + acq_mult * sink_frm->width, RKISP1_CIF_ISP_ACQ_H_SIZE); + rkisp1_write(rkisp1, sink_frm->height, RKISP1_CIF_ISP_ACQ_V_SIZE); + + /* ISP Out Area */ + rkisp1_write(rkisp1, sink_crop->left, RKISP1_CIF_ISP_OUT_H_OFFS); + rkisp1_write(rkisp1, sink_crop->top, RKISP1_CIF_ISP_OUT_V_OFFS); + rkisp1_write(rkisp1, sink_crop->width, RKISP1_CIF_ISP_OUT_H_SIZE); + rkisp1_write(rkisp1, sink_crop->height, RKISP1_CIF_ISP_OUT_V_SIZE); + + irq_mask |= RKISP1_CIF_ISP_FRAME | RKISP1_CIF_ISP_V_START | + RKISP1_CIF_ISP_PIC_SIZE_ERROR | RKISP1_CIF_ISP_FRAME_IN; + rkisp1_write(rkisp1, irq_mask, RKISP1_CIF_ISP_IMSC); + + if (src_fmt->fmt_type == RKISP1_FMT_BAYER) { + rkisp1_params_disable(&rkisp1->params); + } else { + struct v4l2_mbus_framefmt *src_frm; + + src_frm = rkisp1_isp_get_pad_fmt(&rkisp1->isp, NULL, + RKISP1_ISP_PAD_SINK_VIDEO, + V4L2_SUBDEV_FORMAT_ACTIVE); + rkisp1_params_configure(&rkisp1->params, sink_fmt->bayer_pat, + src_frm->quantization); + } + + return 0; +} + +static int rkisp1_config_dvp(struct rkisp1_device *rkisp1) +{ + const struct rkisp1_isp_mbus_info *sink_fmt = rkisp1->isp.sink_fmt; + u32 val, input_sel; + + switch (sink_fmt->bus_width) { + case 8: + input_sel = RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_8B_ZERO; + break; + case 10: + input_sel = RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_10B_ZERO; + break; + case 12: + input_sel = RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_12B; + break; + default: + dev_err(rkisp1->dev, "Invalid bus width\n"); + return -EINVAL; + } + + val = rkisp1_read(rkisp1, RKISP1_CIF_ISP_ACQ_PROP); + rkisp1_write(rkisp1, val | input_sel, RKISP1_CIF_ISP_ACQ_PROP); + + return 0; +} + +static int rkisp1_config_mipi(struct rkisp1_device *rkisp1) +{ + const struct rkisp1_isp_mbus_info *sink_fmt = rkisp1->isp.sink_fmt; + unsigned int lanes; + u32 mipi_ctrl; + + /* + * rkisp1->active_sensor->mbus is set in isp or d-phy notifier_bound + * function + */ + switch (rkisp1->active_sensor->mbus.flags & V4L2_MBUS_CSI2_LANES) { + case V4L2_MBUS_CSI2_4_LANE: + lanes = 4; + break; + case V4L2_MBUS_CSI2_3_LANE: + lanes = 3; + break; + case V4L2_MBUS_CSI2_2_LANE: + lanes = 2; + break; + case V4L2_MBUS_CSI2_1_LANE: + lanes = 1; + break; + default: + return -EINVAL; + } + + mipi_ctrl = RKISP1_CIF_MIPI_CTRL_NUM_LANES(lanes - 1) | + RKISP1_CIF_MIPI_CTRL_SHUTDOWNLANES(0xf) | + RKISP1_CIF_MIPI_CTRL_ERR_SOT_SYNC_HS_SKIP | + RKISP1_CIF_MIPI_CTRL_CLOCKLANE_ENA; + + rkisp1_write(rkisp1, mipi_ctrl, RKISP1_CIF_MIPI_CTRL); + + /* Configure Data Type and Virtual Channel */ + rkisp1_write(rkisp1, + RKISP1_CIF_MIPI_DATA_SEL_DT(sink_fmt->mipi_dt) | + RKISP1_CIF_MIPI_DATA_SEL_VC(0), + RKISP1_CIF_MIPI_IMG_DATA_SEL); + + /* Clear MIPI interrupts */ + rkisp1_write(rkisp1, ~0, RKISP1_CIF_MIPI_ICR); + /* + * Disable RKISP1_CIF_MIPI_ERR_DPHY interrupt here temporary for + * isp bus may be dead when switch isp. + */ + rkisp1_write(rkisp1, + RKISP1_CIF_MIPI_FRAME_END | RKISP1_CIF_MIPI_ERR_CSI | + RKISP1_CIF_MIPI_ERR_DPHY | + RKISP1_CIF_MIPI_SYNC_FIFO_OVFLW(0x03) | + RKISP1_CIF_MIPI_ADD_DATA_OVFLW, + RKISP1_CIF_MIPI_IMSC); + + dev_dbg(rkisp1->dev, "\n MIPI_CTRL 0x%08x\n" + " MIPI_IMG_DATA_SEL 0x%08x\n" + " MIPI_STATUS 0x%08x\n" + " MIPI_IMSC 0x%08x\n", + rkisp1_read(rkisp1, RKISP1_CIF_MIPI_CTRL), + rkisp1_read(rkisp1, RKISP1_CIF_MIPI_IMG_DATA_SEL), + rkisp1_read(rkisp1, RKISP1_CIF_MIPI_STATUS), + rkisp1_read(rkisp1, RKISP1_CIF_MIPI_IMSC)); + + return 0; +} + +/* Configure MUX */ +static int rkisp1_config_path(struct rkisp1_device *rkisp1) +{ + struct rkisp1_sensor_async *sensor = rkisp1->active_sensor; + u32 dpcl = rkisp1_read(rkisp1, RKISP1_CIF_VI_DPCL); + int ret = 0; + + if (sensor->mbus.type == V4L2_MBUS_BT656 || + sensor->mbus.type == V4L2_MBUS_PARALLEL) { + ret = rkisp1_config_dvp(rkisp1); + dpcl |= RKISP1_CIF_VI_DPCL_IF_SEL_PARALLEL; + } else if (sensor->mbus.type == V4L2_MBUS_CSI2_DPHY) { + ret = rkisp1_config_mipi(rkisp1); + dpcl |= RKISP1_CIF_VI_DPCL_IF_SEL_MIPI; + } + + rkisp1_write(rkisp1, dpcl, RKISP1_CIF_VI_DPCL); + + return ret; +} + +/* Hardware configure Entry */ +static int rkisp1_config_cif(struct rkisp1_device *rkisp1) +{ + u32 cif_id; + int ret; + + cif_id = rkisp1_read(rkisp1, RKISP1_CIF_VI_ID); + dev_dbg(rkisp1->dev, "CIF_ID 0x%08x\n", cif_id); + + ret = rkisp1_config_isp(rkisp1); + if (ret) + return ret; + ret = rkisp1_config_path(rkisp1); + if (ret) + return ret; + rkisp1_config_ism(rkisp1); + + return 0; +} + +static int rkisp1_isp_stop(struct rkisp1_device *rkisp1) +{ + u32 val; + + /* + * ISP(mi) stop in mi frame end -> Stop ISP(mipi) -> + * Stop ISP(isp) ->wait for ISP isp off + */ + /* stop and clear MI, MIPI, and ISP interrupts */ + rkisp1_write(rkisp1, 0, RKISP1_CIF_MIPI_IMSC); + rkisp1_write(rkisp1, ~0, RKISP1_CIF_MIPI_ICR); + + rkisp1_write(rkisp1, 0, RKISP1_CIF_ISP_IMSC); + rkisp1_write(rkisp1, ~0, RKISP1_CIF_ISP_ICR); + + rkisp1_write(rkisp1, 0, RKISP1_CIF_MI_IMSC); + rkisp1_write(rkisp1, ~0, RKISP1_CIF_MI_ICR); + val = rkisp1_read(rkisp1, RKISP1_CIF_MIPI_CTRL); + rkisp1_write(rkisp1, val & (~RKISP1_CIF_MIPI_CTRL_OUTPUT_ENA), + RKISP1_CIF_MIPI_CTRL); + /* stop ISP */ + val = rkisp1_read(rkisp1, RKISP1_CIF_ISP_CTRL); + val &= ~(RKISP1_CIF_ISP_CTRL_ISP_INFORM_ENABLE | + RKISP1_CIF_ISP_CTRL_ISP_ENABLE); + rkisp1_write(rkisp1, val, RKISP1_CIF_ISP_CTRL); + + val = rkisp1_read(rkisp1, RKISP1_CIF_ISP_CTRL); + rkisp1_write(rkisp1, val | RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD, + RKISP1_CIF_ISP_CTRL); + + readx_poll_timeout(readl, rkisp1->base_addr + RKISP1_CIF_ISP_RIS, + val, val & RKISP1_CIF_ISP_OFF, 20, 100); + rkisp1_write(rkisp1, + RKISP1_CIF_IRCL_MIPI_SW_RST | RKISP1_CIF_IRCL_ISP_SW_RST, + RKISP1_CIF_IRCL); + rkisp1_write(rkisp1, 0x0, RKISP1_CIF_IRCL); + + return 0; +} + +static void rkisp1_config_clk(struct rkisp1_device *rkisp1) +{ + u32 val = RKISP1_CIF_ICCL_ISP_CLK | RKISP1_CIF_ICCL_CP_CLK | + RKISP1_CIF_ICCL_MRSZ_CLK | RKISP1_CIF_ICCL_SRSZ_CLK | + RKISP1_CIF_ICCL_JPEG_CLK | RKISP1_CIF_ICCL_MI_CLK | + RKISP1_CIF_ICCL_IE_CLK | RKISP1_CIF_ICCL_MIPI_CLK | + RKISP1_CIF_ICCL_DCROP_CLK; + + rkisp1_write(rkisp1, val, RKISP1_CIF_ICCL); +} + +static int rkisp1_isp_start(struct rkisp1_device *rkisp1) +{ + struct rkisp1_sensor_async *sensor = rkisp1->active_sensor; + u32 val; + + rkisp1_config_clk(rkisp1); + + /* Activate MIPI */ + if (sensor->mbus.type == V4L2_MBUS_CSI2_DPHY) { + val = rkisp1_read(rkisp1, RKISP1_CIF_MIPI_CTRL); + rkisp1_write(rkisp1, val | RKISP1_CIF_MIPI_CTRL_OUTPUT_ENA, + RKISP1_CIF_MIPI_CTRL); + } + /* Activate ISP */ + val = rkisp1_read(rkisp1, RKISP1_CIF_ISP_CTRL); + val |= RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD | + RKISP1_CIF_ISP_CTRL_ISP_ENABLE | + RKISP1_CIF_ISP_CTRL_ISP_INFORM_ENABLE; + rkisp1_write(rkisp1, val, RKISP1_CIF_ISP_CTRL); + + /* + * CIF spec says to wait for sufficient time after enabling + * the MIPI interface and before starting the sensor output. + */ + usleep_range(1000, 1200); + + return 0; +} + +/* ---------------------------------------------------------------------------- + * Subdev pad operations + */ + +static int rkisp1_isp_enum_mbus_code(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_mbus_code_enum *code) +{ + unsigned int i, dir; + int pos = 0; + + if (code->pad == RKISP1_ISP_PAD_SINK_VIDEO) { + dir = RKISP1_DIR_SINK; + } else if (code->pad == RKISP1_ISP_PAD_SOURCE_VIDEO) { + dir = RKISP1_DIR_SRC; + } else { + if (code->index > 0) + return -EINVAL; + code->code = MEDIA_BUS_FMT_FIXED; + return 0; + } + + if (code->index >= ARRAY_SIZE(rkisp1_isp_formats)) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(rkisp1_isp_formats); i++) { + const struct rkisp1_isp_mbus_info *fmt = &rkisp1_isp_formats[i]; + + if (fmt->direction & dir) + pos++; + + if (code->index == pos - 1) { + code->code = fmt->mbus_code; + return 0; + } + } + + return -EINVAL; +} + +static int rkisp1_isp_init_config(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg) +{ + struct v4l2_mbus_framefmt *sink_fmt, *src_fmt; + struct v4l2_rect *sink_crop, *src_crop; + + sink_fmt = v4l2_subdev_get_try_format(sd, cfg, + RKISP1_ISP_PAD_SINK_VIDEO); + sink_fmt->width = RKISP1_DEFAULT_WIDTH; + sink_fmt->height = RKISP1_DEFAULT_HEIGHT; + sink_fmt->field = V4L2_FIELD_NONE; + sink_fmt->code = RKISP1_DEF_SINK_PAD_FMT; + + sink_crop = v4l2_subdev_get_try_crop(sd, cfg, + RKISP1_ISP_PAD_SINK_VIDEO); + sink_crop->width = RKISP1_DEFAULT_WIDTH; + sink_crop->height = RKISP1_DEFAULT_HEIGHT; + sink_crop->left = 0; + sink_crop->top = 0; + + src_fmt = v4l2_subdev_get_try_format(sd, cfg, + RKISP1_ISP_PAD_SOURCE_VIDEO); + *src_fmt = *sink_fmt; + src_fmt->code = RKISP1_DEF_SRC_PAD_FMT; + src_fmt->quantization = V4L2_QUANTIZATION_FULL_RANGE; + + src_crop = v4l2_subdev_get_try_crop(sd, cfg, + RKISP1_ISP_PAD_SOURCE_VIDEO); + *src_crop = *sink_crop; + + sink_fmt = v4l2_subdev_get_try_format(sd, cfg, + RKISP1_ISP_PAD_SINK_PARAMS); + src_fmt = v4l2_subdev_get_try_format(sd, cfg, + RKISP1_ISP_PAD_SOURCE_STATS); + sink_fmt->width = RKISP1_DEFAULT_WIDTH; + sink_fmt->height = RKISP1_DEFAULT_HEIGHT; + sink_fmt->field = V4L2_FIELD_NONE; + sink_fmt->code = MEDIA_BUS_FMT_FIXED; + *src_fmt = *sink_fmt; + + return 0; +} + +static void rkisp1_isp_set_src_fmt(struct rkisp1_isp *isp, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_mbus_framefmt *format, + unsigned int which) +{ + const struct rkisp1_isp_mbus_info *mbus_info; + struct v4l2_mbus_framefmt *src_fmt; + const struct v4l2_rect *src_crop; + + src_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, + RKISP1_ISP_PAD_SOURCE_VIDEO, which); + src_crop = rkisp1_isp_get_pad_crop(isp, cfg, + RKISP1_ISP_PAD_SOURCE_VIDEO, which); + + src_fmt->code = format->code; + mbus_info = rkisp1_isp_mbus_info_get(src_fmt->code); + if (!mbus_info) { + src_fmt->code = RKISP1_DEF_SRC_PAD_FMT; + mbus_info = rkisp1_isp_mbus_info_get(src_fmt->code); + } + if (which == V4L2_SUBDEV_FORMAT_ACTIVE) + isp->src_fmt = mbus_info; + src_fmt->width = src_crop->width; + src_fmt->height = src_crop->height; + src_fmt->quantization = format->quantization; + /* full range by default */ + if (!src_fmt->quantization) + src_fmt->quantization = V4L2_QUANTIZATION_FULL_RANGE; + + *format = *src_fmt; +} + +static void rkisp1_isp_set_src_crop(struct rkisp1_isp *isp, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_rect *r, unsigned int which) +{ + struct v4l2_mbus_framefmt *src_fmt; + const struct v4l2_rect *sink_crop; + struct v4l2_rect *src_crop; + + src_crop = rkisp1_isp_get_pad_crop(isp, cfg, + RKISP1_ISP_PAD_SOURCE_VIDEO, + which); + sink_crop = rkisp1_isp_get_pad_crop(isp, cfg, + RKISP1_ISP_PAD_SINK_VIDEO, + which); + + src_crop->left = ALIGN(r->left, 2); + src_crop->width = ALIGN(r->width, 2); + src_crop->top = r->top; + src_crop->height = r->height; + rkisp1_sd_adjust_crop_rect(src_crop, sink_crop); + + *r = *src_crop; + + /* Propagate to out format */ + src_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, + RKISP1_ISP_PAD_SOURCE_VIDEO, which); + rkisp1_isp_set_src_fmt(isp, cfg, src_fmt, which); +} + +static void rkisp1_isp_set_sink_crop(struct rkisp1_isp *isp, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_rect *r, unsigned int which) +{ + struct v4l2_rect *sink_crop, *src_crop; + struct v4l2_mbus_framefmt *sink_fmt; + + sink_crop = rkisp1_isp_get_pad_crop(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + which); + sink_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + which); + + sink_crop->left = ALIGN(r->left, 2); + sink_crop->width = ALIGN(r->width, 2); + sink_crop->top = r->top; + sink_crop->height = r->height; + rkisp1_sd_adjust_crop(sink_crop, sink_fmt); + + *r = *sink_crop; + + /* Propagate to out crop */ + src_crop = rkisp1_isp_get_pad_crop(isp, cfg, + RKISP1_ISP_PAD_SOURCE_VIDEO, which); + rkisp1_isp_set_src_crop(isp, cfg, src_crop, which); +} + +static void rkisp1_isp_set_sink_fmt(struct rkisp1_isp *isp, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_mbus_framefmt *format, + unsigned int which) +{ + const struct rkisp1_isp_mbus_info *mbus_info; + struct v4l2_mbus_framefmt *sink_fmt; + struct v4l2_rect *sink_crop; + + sink_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + which); + sink_fmt->code = format->code; + mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); + if (!mbus_info) { + sink_fmt->code = RKISP1_DEF_SINK_PAD_FMT; + mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); + } + if (which == V4L2_SUBDEV_FORMAT_ACTIVE) + isp->sink_fmt = mbus_info; + + sink_fmt->width = clamp_t(u32, format->width, + RKISP1_ISP_MIN_WIDTH, + RKISP1_ISP_MAX_WIDTH); + sink_fmt->height = clamp_t(u32, format->height, + RKISP1_ISP_MIN_HEIGHT, + RKISP1_ISP_MAX_HEIGHT); + + *format = *sink_fmt; + + /* Propagate to in crop */ + sink_crop = rkisp1_isp_get_pad_crop(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + which); + rkisp1_isp_set_sink_crop(isp, cfg, sink_crop, which); +} + +static int rkisp1_isp_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_format *fmt) +{ + struct rkisp1_isp *isp = container_of(sd, struct rkisp1_isp, sd); + + fmt->format = *rkisp1_isp_get_pad_fmt(isp, cfg, fmt->pad, fmt->which); + return 0; +} + +static int rkisp1_isp_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_format *fmt) +{ + struct rkisp1_isp *isp = container_of(sd, struct rkisp1_isp, sd); + + if (fmt->pad == RKISP1_ISP_PAD_SINK_VIDEO) + rkisp1_isp_set_sink_fmt(isp, cfg, &fmt->format, fmt->which); + else if (fmt->pad == RKISP1_ISP_PAD_SOURCE_VIDEO) + rkisp1_isp_set_src_fmt(isp, cfg, &fmt->format, fmt->which); + else + fmt->format = *rkisp1_isp_get_pad_fmt(isp, cfg, fmt->pad, + fmt->which); + + return 0; +} + +static int rkisp1_isp_get_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_selection *sel) +{ + struct rkisp1_isp *isp = container_of(sd, struct rkisp1_isp, sd); + + if (sel->pad != RKISP1_ISP_PAD_SOURCE_VIDEO && + sel->pad != RKISP1_ISP_PAD_SINK_VIDEO) + return -EINVAL; + + switch (sel->target) { + case V4L2_SEL_TGT_CROP_BOUNDS: + if (sel->pad == RKISP1_ISP_PAD_SINK_VIDEO) { + struct v4l2_mbus_framefmt *fmt; + + fmt = rkisp1_isp_get_pad_fmt(isp, cfg, sel->pad, + sel->which); + sel->r.height = fmt->height; + sel->r.width = fmt->width; + sel->r.left = 0; + sel->r.top = 0; + } else { + sel->r = *rkisp1_isp_get_pad_crop(isp, cfg, + RKISP1_ISP_PAD_SINK_VIDEO, + sel->which); + } + break; + case V4L2_SEL_TGT_CROP: + sel->r = *rkisp1_isp_get_pad_crop(isp, cfg, sel->pad, + sel->which); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int rkisp1_isp_set_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_selection *sel) +{ + struct rkisp1_device *rkisp1 = + container_of(sd->v4l2_dev, struct rkisp1_device, v4l2_dev); + struct rkisp1_isp *isp = container_of(sd, struct rkisp1_isp, sd); + + if (sel->target != V4L2_SEL_TGT_CROP) + return -EINVAL; + + dev_dbg(rkisp1->dev, "%s: pad: %d sel(%d,%d)/%dx%d\n", __func__, + sel->pad, sel->r.left, sel->r.top, sel->r.width, sel->r.height); + + if (sel->pad == RKISP1_ISP_PAD_SINK_VIDEO) + rkisp1_isp_set_sink_crop(isp, cfg, &sel->r, sel->which); + else if (sel->pad == RKISP1_ISP_PAD_SOURCE_VIDEO) + rkisp1_isp_set_src_crop(isp, cfg, &sel->r, sel->which); + else + return -EINVAL; + + return 0; +} + +static int rkisp1_subdev_link_validate(struct media_link *link) +{ + if (link->sink->index == RKISP1_ISP_PAD_SINK_PARAMS) + return 0; + + return v4l2_subdev_link_validate(link); +} + +static const struct v4l2_subdev_pad_ops rkisp1_isp_pad_ops = { + .enum_mbus_code = rkisp1_isp_enum_mbus_code, + .get_selection = rkisp1_isp_get_selection, + .set_selection = rkisp1_isp_set_selection, + .init_cfg = rkisp1_isp_init_config, + .get_fmt = rkisp1_isp_get_fmt, + .set_fmt = rkisp1_isp_set_fmt, + .link_validate = v4l2_subdev_link_validate_default, +}; + +/* ---------------------------------------------------------------------------- + * Stream operations + */ + +static int rkisp1_mipi_csi2_start(struct rkisp1_isp *isp, + struct rkisp1_sensor_async *sensor) +{ + union phy_configure_opts opts; + struct phy_configure_opts_mipi_dphy *cfg = &opts.mipi_dphy; + s64 pixel_clock; + + if (!sensor->pixel_rate_ctrl) { + dev_warn(sensor->sd->dev, "No pixel rate control in subdev\n"); + return -EPIPE; + } + + pixel_clock = v4l2_ctrl_g_ctrl_int64(sensor->pixel_rate_ctrl); + if (!pixel_clock) { + dev_err(sensor->sd->dev, "Invalid pixel rate value\n"); + return -EINVAL; + } + + phy_mipi_dphy_get_default_config(pixel_clock, isp->sink_fmt->bus_width, + sensor->lanes, cfg); + phy_set_mode(sensor->dphy, PHY_MODE_MIPI_DPHY); + phy_configure(sensor->dphy, &opts); + phy_power_on(sensor->dphy); + + return 0; +} + +static void rkisp1_mipi_csi2_stop(struct rkisp1_sensor_async *sensor) +{ + phy_power_off(sensor->dphy); +} + +static int rkisp1_isp_s_stream(struct v4l2_subdev *sd, int enable) +{ + struct rkisp1_device *rkisp1 = + container_of(sd->v4l2_dev, struct rkisp1_device, v4l2_dev); + struct v4l2_subdev *sensor_sd; + int ret = 0; + + if (!enable) { + ret = rkisp1_isp_stop(rkisp1); + if (ret) + return ret; + rkisp1_mipi_csi2_stop(rkisp1->active_sensor); + return 0; + } + + sensor_sd = rkisp1_get_remote_sensor(sd); + if (!sensor_sd) + return -ENODEV; + rkisp1->active_sensor = container_of(sensor_sd->asd, + struct rkisp1_sensor_async, asd); + + atomic_set(&rkisp1->isp.frame_sequence, -1); + ret = rkisp1_config_cif(rkisp1); + if (ret) + return ret; + + if (rkisp1->active_sensor->mbus.type != V4L2_MBUS_CSI2_DPHY) + return -EINVAL; + + ret = rkisp1_mipi_csi2_start(&rkisp1->isp, rkisp1->active_sensor); + if (ret) + return ret; + + ret = rkisp1_isp_start(rkisp1); + if (ret) + rkisp1_mipi_csi2_stop(rkisp1->active_sensor); + + return ret; +} + +static int rkisp1_isp_subs_evt(struct v4l2_subdev *sd, struct v4l2_fh *fh, + struct v4l2_event_subscription *sub) +{ + if (sub->type != V4L2_EVENT_FRAME_SYNC) + return -EINVAL; + + /* V4L2_EVENT_FRAME_SYNC doesn't require an id, so zero should be set */ + if (sub->id != 0) + return -EINVAL; + + return v4l2_event_subscribe(fh, sub, 0, NULL); +} + +static const struct media_entity_operations rkisp1_isp_media_ops = { + .link_validate = rkisp1_subdev_link_validate, +}; + +static const struct v4l2_subdev_video_ops rkisp1_isp_video_ops = { + .s_stream = rkisp1_isp_s_stream, +}; + +static const struct v4l2_subdev_core_ops rkisp1_isp_core_ops = { + .subscribe_event = rkisp1_isp_subs_evt, + .unsubscribe_event = v4l2_event_subdev_unsubscribe, +}; + +static const struct v4l2_subdev_ops rkisp1_isp_ops = { + .core = &rkisp1_isp_core_ops, + .video = &rkisp1_isp_video_ops, + .pad = &rkisp1_isp_pad_ops, +}; + +int rkisp1_isp_register(struct rkisp1_device *rkisp1, + struct v4l2_device *v4l2_dev) +{ + struct rkisp1_isp *isp = &rkisp1->isp; + struct media_pad *pads = isp->pads; + struct v4l2_subdev *sd = &isp->sd; + int ret; + + v4l2_subdev_init(sd, &rkisp1_isp_ops); + sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS; + sd->entity.ops = &rkisp1_isp_media_ops; + sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER; + sd->owner = THIS_MODULE; + strscpy(sd->name, RKISP1_ISP_DEV_NAME, sizeof(sd->name)); + + pads[RKISP1_ISP_PAD_SINK_VIDEO].flags = MEDIA_PAD_FL_SINK | + MEDIA_PAD_FL_MUST_CONNECT; + pads[RKISP1_ISP_PAD_SINK_PARAMS].flags = MEDIA_PAD_FL_SINK; + pads[RKISP1_ISP_PAD_SOURCE_VIDEO].flags = MEDIA_PAD_FL_SOURCE; + pads[RKISP1_ISP_PAD_SOURCE_STATS].flags = MEDIA_PAD_FL_SOURCE; + + isp->sink_fmt = rkisp1_isp_mbus_info_get(RKISP1_DEF_SINK_PAD_FMT); + isp->src_fmt = rkisp1_isp_mbus_info_get(RKISP1_DEF_SRC_PAD_FMT); + + ret = media_entity_pads_init(&sd->entity, RKISP1_ISP_PAD_MAX, pads); + if (ret) + return ret; + + ret = v4l2_device_register_subdev(v4l2_dev, sd); + if (ret) { + dev_err(sd->dev, "Failed to register isp subdev\n"); + goto err_cleanup_media_entity; + } + + rkisp1_isp_init_config(sd, rkisp1->isp.pad_cfg); + return 0; + +err_cleanup_media_entity: + media_entity_cleanup(&sd->entity); + + return ret; +} + +void rkisp1_isp_unregister(struct rkisp1_device *rkisp1) +{ + struct v4l2_subdev *sd = &rkisp1->isp.sd; + + v4l2_device_unregister_subdev(sd); + media_entity_cleanup(&sd->entity); +} + +/* ---------------------------------------------------------------------------- + * Interrupt handlers + */ + +void rkisp1_mipi_isr(struct rkisp1_device *rkisp1) +{ + u32 val, status; + + status = rkisp1_read(rkisp1, RKISP1_CIF_MIPI_MIS); + if (!status) + return; + + rkisp1_write(rkisp1, status, RKISP1_CIF_MIPI_ICR); + + /* + * Disable DPHY errctrl interrupt, because this dphy + * erctrl signal is asserted until the next changes + * of line state. This time is may be too long and cpu + * is hold in this interrupt. + */ + if (status & RKISP1_CIF_MIPI_ERR_CTRL(0x0f)) { + val = rkisp1_read(rkisp1, RKISP1_CIF_MIPI_IMSC); + rkisp1_write(rkisp1, val & ~RKISP1_CIF_MIPI_ERR_CTRL(0x0f), + RKISP1_CIF_MIPI_IMSC); + rkisp1->isp.is_dphy_errctrl_disabled = true; + } + + /* + * Enable DPHY errctrl interrupt again, if mipi have receive + * the whole frame without any error. + */ + if (status == RKISP1_CIF_MIPI_FRAME_END) { + /* + * Enable DPHY errctrl interrupt again, if mipi have receive + * the whole frame without any error. + */ + if (rkisp1->isp.is_dphy_errctrl_disabled) { + val = rkisp1_read(rkisp1, RKISP1_CIF_MIPI_IMSC); + val |= RKISP1_CIF_MIPI_ERR_CTRL(0x0f); + rkisp1_write(rkisp1, val, RKISP1_CIF_MIPI_IMSC); + rkisp1->isp.is_dphy_errctrl_disabled = false; + } + } else { + rkisp1->debug.mipi_error++; + } +} + +static void rkisp1_isp_queue_event_sof(struct rkisp1_isp *isp) +{ + struct v4l2_event event = { + .type = V4L2_EVENT_FRAME_SYNC, + }; + + /* + * Increment the frame sequence on the vsync signal. + * This will allow applications to detect dropped. + * Note that there is a debugfs counter for dropped + * frames, but using this event is more accurate. + */ + event.u.frame_sync.frame_sequence = + atomic_inc_return(&isp->frame_sequence); + v4l2_event_queue(isp->sd.devnode, &event); +} + +void rkisp1_isp_isr(struct rkisp1_device *rkisp1) +{ + u32 status, isp_err; + + status = rkisp1_read(rkisp1, RKISP1_CIF_ISP_MIS); + if (!status) + return; + + rkisp1_write(rkisp1, status, RKISP1_CIF_ISP_ICR); + + /* Vertical sync signal, starting generating new frame */ + if (status & RKISP1_CIF_ISP_V_START) + rkisp1_isp_queue_event_sof(&rkisp1->isp); + + if (status & RKISP1_CIF_ISP_PIC_SIZE_ERROR) { + /* Clear pic_size_error */ + isp_err = rkisp1_read(rkisp1, RKISP1_CIF_ISP_ERR); + rkisp1_write(rkisp1, isp_err, RKISP1_CIF_ISP_ERR_CLR); + rkisp1->debug.pic_size_error++; + } else if (status & RKISP1_CIF_ISP_DATA_LOSS) { + /* keep track of data_loss in debugfs */ + rkisp1->debug.data_loss++; + } + + if (status & RKISP1_CIF_ISP_FRAME) { + u32 isp_ris; + + /* New frame from the sensor received */ + isp_ris = rkisp1_read(rkisp1, RKISP1_CIF_ISP_RIS); + if (isp_ris & (RKISP1_CIF_ISP_AWB_DONE | + RKISP1_CIF_ISP_AFM_FIN | + RKISP1_CIF_ISP_EXP_END | + RKISP1_CIF_ISP_HIST_MEASURE_RDY)) + rkisp1_stats_isr(&rkisp1->stats, isp_ris); + } + + /* + * Then update changed configs. Some of them involve + * lot of register writes. Do those only one per frame. + * Do the updates in the order of the processing flow. + */ + rkisp1_params_isr(rkisp1, status); +} diff --git a/drivers/staging/media/rkisp1/rkisp1-params.c b/drivers/staging/media/rkisp1/rkisp1-params.c new file mode 100644 index 000000000000..781f0ca85af1 --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-params.c @@ -0,0 +1,1630 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip ISP1 Driver - Params subdevice + * + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#include <media/v4l2-common.h> +#include <media/v4l2-event.h> +#include <media/v4l2-ioctl.h> +#include <media/videobuf2-core.h> +#include <media/videobuf2-vmalloc.h> /* for ISP params */ + +#include "rkisp1-common.h" + +#define RKISP1_PARAMS_DEV_NAME RKISP1_DRIVER_NAME "_params" + +#define RKISP1_ISP_PARAMS_REQ_BUFS_MIN 2 +#define RKISP1_ISP_PARAMS_REQ_BUFS_MAX 8 + +#define RKISP1_ISP_DPCC_LINE_THRESH(n) \ + (RKISP1_CIF_ISP_DPCC_LINE_THRESH_1 + 0x14 * (n)) +#define RKISP1_ISP_DPCC_LINE_MAD_FAC(n) \ + (RKISP1_CIF_ISP_DPCC_LINE_MAD_FAC_1 + 0x14 * (n)) +#define RKISP1_ISP_DPCC_PG_FAC(n) \ + (RKISP1_CIF_ISP_DPCC_PG_FAC_1 + 0x14 * (n)) +#define RKISP1_ISP_DPCC_RND_THRESH(n) \ + (RKISP1_CIF_ISP_DPCC_RND_THRESH_1 + 0x14 * (n)) +#define RKISP1_ISP_DPCC_RG_FAC(n) \ + (RKISP1_CIF_ISP_DPCC_RG_FAC_1 + 0x14 * (n)) +#define RKISP1_ISP_CC_COEFF(n) \ + (RKISP1_CIF_ISP_CC_COEFF_0 + (n) * 4) + +static inline void +rkisp1_param_set_bits(struct rkisp1_params *params, u32 reg, u32 bit_mask) +{ + u32 val; + + val = rkisp1_read(params->rkisp1, reg); + rkisp1_write(params->rkisp1, val | bit_mask, reg); +} + +static inline void +rkisp1_param_clear_bits(struct rkisp1_params *params, u32 reg, u32 bit_mask) +{ + u32 val; + + val = rkisp1_read(params->rkisp1, reg); + rkisp1_write(params->rkisp1, val & ~bit_mask, reg); +} + +/* ISP BP interface function */ +static void rkisp1_dpcc_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_dpcc_config *arg) +{ + unsigned int i; + u32 mode; + + /* avoid to override the old enable value */ + mode = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_DPCC_MODE); + mode &= RKISP1_CIF_ISP_DPCC_ENA; + mode |= arg->mode & ~RKISP1_CIF_ISP_DPCC_ENA; + rkisp1_write(params->rkisp1, mode, RKISP1_CIF_ISP_DPCC_MODE); + rkisp1_write(params->rkisp1, arg->output_mode, + RKISP1_CIF_ISP_DPCC_OUTPUT_MODE); + rkisp1_write(params->rkisp1, arg->set_use, + RKISP1_CIF_ISP_DPCC_SET_USE); + + rkisp1_write(params->rkisp1, arg->methods[0].method, + RKISP1_CIF_ISP_DPCC_METHODS_SET_1); + rkisp1_write(params->rkisp1, arg->methods[1].method, + RKISP1_CIF_ISP_DPCC_METHODS_SET_2); + rkisp1_write(params->rkisp1, arg->methods[2].method, + RKISP1_CIF_ISP_DPCC_METHODS_SET_3); + for (i = 0; i < RKISP1_CIF_ISP_DPCC_METHODS_MAX; i++) { + rkisp1_write(params->rkisp1, arg->methods[i].line_thresh, + RKISP1_ISP_DPCC_LINE_THRESH(i)); + rkisp1_write(params->rkisp1, arg->methods[i].line_mad_fac, + RKISP1_ISP_DPCC_LINE_MAD_FAC(i)); + rkisp1_write(params->rkisp1, arg->methods[i].pg_fac, + RKISP1_ISP_DPCC_PG_FAC(i)); + rkisp1_write(params->rkisp1, arg->methods[i].rnd_thresh, + RKISP1_ISP_DPCC_RND_THRESH(i)); + rkisp1_write(params->rkisp1, arg->methods[i].rg_fac, + RKISP1_ISP_DPCC_RG_FAC(i)); + } + + rkisp1_write(params->rkisp1, arg->rnd_offs, + RKISP1_CIF_ISP_DPCC_RND_OFFS); + rkisp1_write(params->rkisp1, arg->ro_limits, + RKISP1_CIF_ISP_DPCC_RO_LIMITS); +} + +/* ISP black level subtraction interface function */ +static void rkisp1_bls_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_bls_config *arg) +{ + /* avoid to override the old enable value */ + u32 new_control; + + new_control = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_BLS_CTRL); + new_control &= RKISP1_CIF_ISP_BLS_ENA; + /* fixed subtraction values */ + if (!arg->enable_auto) { + const struct rkisp1_cif_isp_bls_fixed_val *pval = + &arg->fixed_val; + + switch (params->raw_type) { + case RKISP1_RAW_BGGR: + rkisp1_write(params->rkisp1, + pval->r, RKISP1_CIF_ISP_BLS_D_FIXED); + rkisp1_write(params->rkisp1, + pval->gr, RKISP1_CIF_ISP_BLS_C_FIXED); + rkisp1_write(params->rkisp1, + pval->gb, RKISP1_CIF_ISP_BLS_B_FIXED); + rkisp1_write(params->rkisp1, + pval->b, RKISP1_CIF_ISP_BLS_A_FIXED); + break; + case RKISP1_RAW_GBRG: + rkisp1_write(params->rkisp1, + pval->r, RKISP1_CIF_ISP_BLS_C_FIXED); + rkisp1_write(params->rkisp1, + pval->gr, RKISP1_CIF_ISP_BLS_D_FIXED); + rkisp1_write(params->rkisp1, + pval->gb, RKISP1_CIF_ISP_BLS_A_FIXED); + rkisp1_write(params->rkisp1, + pval->b, RKISP1_CIF_ISP_BLS_B_FIXED); + break; + case RKISP1_RAW_GRBG: + rkisp1_write(params->rkisp1, + pval->r, RKISP1_CIF_ISP_BLS_B_FIXED); + rkisp1_write(params->rkisp1, + pval->gr, RKISP1_CIF_ISP_BLS_A_FIXED); + rkisp1_write(params->rkisp1, + pval->gb, RKISP1_CIF_ISP_BLS_D_FIXED); + rkisp1_write(params->rkisp1, + pval->b, RKISP1_CIF_ISP_BLS_C_FIXED); + break; + case RKISP1_RAW_RGGB: + rkisp1_write(params->rkisp1, + pval->r, RKISP1_CIF_ISP_BLS_A_FIXED); + rkisp1_write(params->rkisp1, + pval->gr, RKISP1_CIF_ISP_BLS_B_FIXED); + rkisp1_write(params->rkisp1, + pval->gb, RKISP1_CIF_ISP_BLS_C_FIXED); + rkisp1_write(params->rkisp1, + pval->b, RKISP1_CIF_ISP_BLS_D_FIXED); + break; + default: + break; + } + + } else { + if (arg->en_windows & BIT(1)) { + rkisp1_write(params->rkisp1, arg->bls_window2.h_offs, + RKISP1_CIF_ISP_BLS_H2_START); + rkisp1_write(params->rkisp1, arg->bls_window2.h_size, + RKISP1_CIF_ISP_BLS_H2_STOP); + rkisp1_write(params->rkisp1, arg->bls_window2.v_offs, + RKISP1_CIF_ISP_BLS_V2_START); + rkisp1_write(params->rkisp1, arg->bls_window2.v_size, + RKISP1_CIF_ISP_BLS_V2_STOP); + new_control |= RKISP1_CIF_ISP_BLS_WINDOW_2; + } + + if (arg->en_windows & BIT(0)) { + rkisp1_write(params->rkisp1, arg->bls_window1.h_offs, + RKISP1_CIF_ISP_BLS_H1_START); + rkisp1_write(params->rkisp1, arg->bls_window1.h_size, + RKISP1_CIF_ISP_BLS_H1_STOP); + rkisp1_write(params->rkisp1, arg->bls_window1.v_offs, + RKISP1_CIF_ISP_BLS_V1_START); + rkisp1_write(params->rkisp1, arg->bls_window1.v_size, + RKISP1_CIF_ISP_BLS_V1_STOP); + new_control |= RKISP1_CIF_ISP_BLS_WINDOW_1; + } + + rkisp1_write(params->rkisp1, arg->bls_samples, + RKISP1_CIF_ISP_BLS_SAMPLES); + + new_control |= RKISP1_CIF_ISP_BLS_MODE_MEASURED; + } + rkisp1_write(params->rkisp1, new_control, RKISP1_CIF_ISP_BLS_CTRL); +} + +/* ISP LS correction interface function */ +static void +rkisp1_lsc_correct_matrix_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_lsc_config *pconfig) +{ + unsigned int isp_lsc_status, sram_addr, isp_lsc_table_sel, i, j, data; + + isp_lsc_status = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_LSC_STATUS); + + /* RKISP1_CIF_ISP_LSC_TABLE_ADDRESS_153 = ( 17 * 18 ) >> 1 */ + sram_addr = (isp_lsc_status & RKISP1_CIF_ISP_LSC_ACTIVE_TABLE) ? + RKISP1_CIF_ISP_LSC_TABLE_ADDRESS_0 : + RKISP1_CIF_ISP_LSC_TABLE_ADDRESS_153; + rkisp1_write(params->rkisp1, sram_addr, + RKISP1_CIF_ISP_LSC_R_TABLE_ADDR); + rkisp1_write(params->rkisp1, sram_addr, + RKISP1_CIF_ISP_LSC_GR_TABLE_ADDR); + rkisp1_write(params->rkisp1, sram_addr, + RKISP1_CIF_ISP_LSC_GB_TABLE_ADDR); + rkisp1_write(params->rkisp1, sram_addr, + RKISP1_CIF_ISP_LSC_B_TABLE_ADDR); + + /* program data tables (table size is 9 * 17 = 153) */ + for (i = 0; + i < RKISP1_CIF_ISP_LSC_SECTORS_MAX * RKISP1_CIF_ISP_LSC_SECTORS_MAX; + i += RKISP1_CIF_ISP_LSC_SECTORS_MAX) { + /* + * 17 sectors with 2 values in one DWORD = 9 + * DWORDs (2nd value of last DWORD unused) + */ + for (j = 0; j < RKISP1_CIF_ISP_LSC_SECTORS_MAX - 1; j += 2) { + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->r_data_tbl[i + j], + pconfig->r_data_tbl[i + j + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_R_TABLE_DATA); + + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gr_data_tbl[i + j], + pconfig->gr_data_tbl[i + j + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_GR_TABLE_DATA); + + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gb_data_tbl[i + j], + pconfig->gb_data_tbl[i + j + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_GB_TABLE_DATA); + + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->b_data_tbl[i + j], + pconfig->b_data_tbl[i + j + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_B_TABLE_DATA); + } + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->r_data_tbl[i + j], 0); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_R_TABLE_DATA); + + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gr_data_tbl[i + j], 0); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_GR_TABLE_DATA); + + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gb_data_tbl[i + j], 0); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_GB_TABLE_DATA); + + data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->b_data_tbl[i + j], 0); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_B_TABLE_DATA); + } + isp_lsc_table_sel = (isp_lsc_status & RKISP1_CIF_ISP_LSC_ACTIVE_TABLE) ? + RKISP1_CIF_ISP_LSC_TABLE_0 : + RKISP1_CIF_ISP_LSC_TABLE_1; + rkisp1_write(params->rkisp1, isp_lsc_table_sel, + RKISP1_CIF_ISP_LSC_TABLE_SEL); +} + +static void rkisp1_lsc_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_lsc_config *arg) +{ + unsigned int i, data; + u32 lsc_ctrl; + + /* To config must be off , store the current status firstly */ + lsc_ctrl = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_LSC_CTRL); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_LSC_CTRL, + RKISP1_CIF_ISP_LSC_CTRL_ENA); + rkisp1_lsc_correct_matrix_config(params, arg); + + for (i = 0; i < 4; i++) { + /* program x size tables */ + data = RKISP1_CIF_ISP_LSC_SECT_SIZE(arg->x_size_tbl[i * 2], + arg->x_size_tbl[i * 2 + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_XSIZE_01 + i * 4); + + /* program x grad tables */ + data = RKISP1_CIF_ISP_LSC_SECT_SIZE(arg->x_grad_tbl[i * 2], + arg->x_grad_tbl[i * 2 + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_XGRAD_01 + i * 4); + + /* program y size tables */ + data = RKISP1_CIF_ISP_LSC_SECT_SIZE(arg->y_size_tbl[i * 2], + arg->y_size_tbl[i * 2 + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_YSIZE_01 + i * 4); + + /* program y grad tables */ + data = RKISP1_CIF_ISP_LSC_SECT_SIZE(arg->y_grad_tbl[i * 2], + arg->y_grad_tbl[i * 2 + 1]); + rkisp1_write(params->rkisp1, data, + RKISP1_CIF_ISP_LSC_YGRAD_01 + i * 4); + } + + /* restore the lsc ctrl status */ + if (lsc_ctrl & RKISP1_CIF_ISP_LSC_CTRL_ENA) { + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_LSC_CTRL, + RKISP1_CIF_ISP_LSC_CTRL_ENA); + } else { + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_LSC_CTRL, + RKISP1_CIF_ISP_LSC_CTRL_ENA); + } +} + +/* ISP Filtering function */ +static void rkisp1_flt_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_flt_config *arg) +{ + u32 filt_mode; + + rkisp1_write(params->rkisp1, + arg->thresh_bl0, RKISP1_CIF_ISP_FILT_THRESH_BL0); + rkisp1_write(params->rkisp1, + arg->thresh_bl1, RKISP1_CIF_ISP_FILT_THRESH_BL1); + rkisp1_write(params->rkisp1, + arg->thresh_sh0, RKISP1_CIF_ISP_FILT_THRESH_SH0); + rkisp1_write(params->rkisp1, + arg->thresh_sh1, RKISP1_CIF_ISP_FILT_THRESH_SH1); + rkisp1_write(params->rkisp1, arg->fac_bl0, RKISP1_CIF_ISP_FILT_FAC_BL0); + rkisp1_write(params->rkisp1, arg->fac_bl1, RKISP1_CIF_ISP_FILT_FAC_BL1); + rkisp1_write(params->rkisp1, arg->fac_mid, RKISP1_CIF_ISP_FILT_FAC_MID); + rkisp1_write(params->rkisp1, arg->fac_sh0, RKISP1_CIF_ISP_FILT_FAC_SH0); + rkisp1_write(params->rkisp1, arg->fac_sh1, RKISP1_CIF_ISP_FILT_FAC_SH1); + rkisp1_write(params->rkisp1, + arg->lum_weight, RKISP1_CIF_ISP_FILT_LUM_WEIGHT); + + rkisp1_write(params->rkisp1, + (arg->mode ? RKISP1_CIF_ISP_FLT_MODE_DNR : 0) | + RKISP1_CIF_ISP_FLT_CHROMA_V_MODE(arg->chr_v_mode) | + RKISP1_CIF_ISP_FLT_CHROMA_H_MODE(arg->chr_h_mode) | + RKISP1_CIF_ISP_FLT_GREEN_STAGE1(arg->grn_stage1), + RKISP1_CIF_ISP_FILT_MODE); + + /* avoid to override the old enable value */ + filt_mode = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_FILT_MODE); + filt_mode &= RKISP1_CIF_ISP_FLT_ENA; + if (arg->mode) + filt_mode |= RKISP1_CIF_ISP_FLT_MODE_DNR; + filt_mode |= RKISP1_CIF_ISP_FLT_CHROMA_V_MODE(arg->chr_v_mode) | + RKISP1_CIF_ISP_FLT_CHROMA_H_MODE(arg->chr_h_mode) | + RKISP1_CIF_ISP_FLT_GREEN_STAGE1(arg->grn_stage1); + rkisp1_write(params->rkisp1, filt_mode, RKISP1_CIF_ISP_FILT_MODE); +} + +/* ISP demosaic interface function */ +static int rkisp1_bdm_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_bdm_config *arg) +{ + u32 bdm_th; + + /* avoid to override the old enable value */ + bdm_th = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_DEMOSAIC); + bdm_th &= RKISP1_CIF_ISP_DEMOSAIC_BYPASS; + bdm_th |= arg->demosaic_th & ~RKISP1_CIF_ISP_DEMOSAIC_BYPASS; + /* set demosaic threshold */ + rkisp1_write(params->rkisp1, bdm_th, RKISP1_CIF_ISP_DEMOSAIC); + return 0; +} + +/* ISP GAMMA correction interface function */ +static void rkisp1_sdg_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_sdg_config *arg) +{ + unsigned int i; + + rkisp1_write(params->rkisp1, + arg->xa_pnts.gamma_dx0, RKISP1_CIF_ISP_GAMMA_DX_LO); + rkisp1_write(params->rkisp1, + arg->xa_pnts.gamma_dx1, RKISP1_CIF_ISP_GAMMA_DX_HI); + + for (i = 0; i < RKISP1_CIF_ISP_DEGAMMA_CURVE_SIZE; i++) { + rkisp1_write(params->rkisp1, arg->curve_r.gamma_y[i], + RKISP1_CIF_ISP_GAMMA_R_Y0 + i * 4); + rkisp1_write(params->rkisp1, arg->curve_g.gamma_y[i], + RKISP1_CIF_ISP_GAMMA_G_Y0 + i * 4); + rkisp1_write(params->rkisp1, arg->curve_b.gamma_y[i], + RKISP1_CIF_ISP_GAMMA_B_Y0 + i * 4); + } +} + +/* ISP GAMMA correction interface function */ +static void rkisp1_goc_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_goc_config *arg) +{ + unsigned int i; + + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_GAMMA_OUT_ENA); + rkisp1_write(params->rkisp1, arg->mode, RKISP1_CIF_ISP_GAMMA_OUT_MODE); + + for (i = 0; i < RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES; i++) + rkisp1_write(params->rkisp1, arg->gamma_y[i], + RKISP1_CIF_ISP_GAMMA_OUT_Y_0 + i * 4); +} + +/* ISP Cross Talk */ +static void rkisp1_ctk_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_ctk_config *arg) +{ + rkisp1_write(params->rkisp1, arg->coeff0, RKISP1_CIF_ISP_CT_COEFF_0); + rkisp1_write(params->rkisp1, arg->coeff1, RKISP1_CIF_ISP_CT_COEFF_1); + rkisp1_write(params->rkisp1, arg->coeff2, RKISP1_CIF_ISP_CT_COEFF_2); + rkisp1_write(params->rkisp1, arg->coeff3, RKISP1_CIF_ISP_CT_COEFF_3); + rkisp1_write(params->rkisp1, arg->coeff4, RKISP1_CIF_ISP_CT_COEFF_4); + rkisp1_write(params->rkisp1, arg->coeff5, RKISP1_CIF_ISP_CT_COEFF_5); + rkisp1_write(params->rkisp1, arg->coeff6, RKISP1_CIF_ISP_CT_COEFF_6); + rkisp1_write(params->rkisp1, arg->coeff7, RKISP1_CIF_ISP_CT_COEFF_7); + rkisp1_write(params->rkisp1, arg->coeff8, RKISP1_CIF_ISP_CT_COEFF_8); + rkisp1_write(params->rkisp1, arg->ct_offset_r, + RKISP1_CIF_ISP_CT_OFFSET_R); + rkisp1_write(params->rkisp1, arg->ct_offset_g, + RKISP1_CIF_ISP_CT_OFFSET_G); + rkisp1_write(params->rkisp1, arg->ct_offset_b, + RKISP1_CIF_ISP_CT_OFFSET_B); +} + +static void rkisp1_ctk_enable(struct rkisp1_params *params, bool en) +{ + if (en) + return; + + /* Write back the default values. */ + rkisp1_write(params->rkisp1, 0x80, RKISP1_CIF_ISP_CT_COEFF_0); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_COEFF_1); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_COEFF_2); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_COEFF_3); + rkisp1_write(params->rkisp1, 0x80, RKISP1_CIF_ISP_CT_COEFF_4); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_COEFF_5); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_COEFF_6); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_COEFF_7); + rkisp1_write(params->rkisp1, 0x80, RKISP1_CIF_ISP_CT_COEFF_8); + + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_OFFSET_R); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_OFFSET_G); + rkisp1_write(params->rkisp1, 0, RKISP1_CIF_ISP_CT_OFFSET_B); +} + +/* ISP White Balance Mode */ +static void rkisp1_awb_meas_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_awb_meas_config *arg) +{ + u32 reg_val = 0; + /* based on the mode,configure the awb module */ + if (arg->awb_mode == RKISP1_CIF_ISP_AWB_MODE_YCBCR) { + /* Reference Cb and Cr */ + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_AWB_REF_CR_SET(arg->awb_ref_cr) | + arg->awb_ref_cb, RKISP1_CIF_ISP_AWB_REF); + /* Yc Threshold */ + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_AWB_MAX_Y_SET(arg->max_y) | + RKISP1_CIF_ISP_AWB_MIN_Y_SET(arg->min_y) | + RKISP1_CIF_ISP_AWB_MAX_CS_SET(arg->max_csum) | + arg->min_c, RKISP1_CIF_ISP_AWB_THRESH); + } + + reg_val = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_AWB_PROP); + if (arg->enable_ymax_cmp) + reg_val |= RKISP1_CIF_ISP_AWB_YMAX_CMP_EN; + else + reg_val &= ~RKISP1_CIF_ISP_AWB_YMAX_CMP_EN; + rkisp1_write(params->rkisp1, reg_val, RKISP1_CIF_ISP_AWB_PROP); + + /* window offset */ + rkisp1_write(params->rkisp1, + arg->awb_wnd.v_offs, RKISP1_CIF_ISP_AWB_WND_V_OFFS); + rkisp1_write(params->rkisp1, + arg->awb_wnd.h_offs, RKISP1_CIF_ISP_AWB_WND_H_OFFS); + /* AWB window size */ + rkisp1_write(params->rkisp1, + arg->awb_wnd.v_size, RKISP1_CIF_ISP_AWB_WND_V_SIZE); + rkisp1_write(params->rkisp1, + arg->awb_wnd.h_size, RKISP1_CIF_ISP_AWB_WND_H_SIZE); + /* Number of frames */ + rkisp1_write(params->rkisp1, + arg->frames, RKISP1_CIF_ISP_AWB_FRAMES); +} + +static void +rkisp1_awb_meas_enable(struct rkisp1_params *params, + const struct rkisp1_cif_isp_awb_meas_config *arg, + bool en) +{ + u32 reg_val = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_AWB_PROP); + + /* switch off */ + reg_val &= RKISP1_CIF_ISP_AWB_MODE_MASK_NONE; + + if (en) { + if (arg->awb_mode == RKISP1_CIF_ISP_AWB_MODE_RGB) + reg_val |= RKISP1_CIF_ISP_AWB_MODE_RGB_EN; + else + reg_val |= RKISP1_CIF_ISP_AWB_MODE_YCBCR_EN; + + rkisp1_write(params->rkisp1, reg_val, RKISP1_CIF_ISP_AWB_PROP); + + /* Measurements require AWB block be active. */ + rkisp1_param_set_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_AWB_ENA); + } else { + rkisp1_write(params->rkisp1, + reg_val, RKISP1_CIF_ISP_AWB_PROP); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_AWB_ENA); + } +} + +static void +rkisp1_awb_gain_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_awb_gain_config *arg) +{ + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_AWB_GAIN_R_SET(arg->gain_green_r) | + arg->gain_green_b, RKISP1_CIF_ISP_AWB_GAIN_G); + + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_AWB_GAIN_R_SET(arg->gain_red) | + arg->gain_blue, RKISP1_CIF_ISP_AWB_GAIN_RB); +} + +static void rkisp1_aec_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_aec_config *arg) +{ + unsigned int block_hsize, block_vsize; + u32 exp_ctrl; + + /* avoid to override the old enable value */ + exp_ctrl = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_EXP_CTRL); + exp_ctrl &= RKISP1_CIF_ISP_EXP_ENA; + if (arg->autostop) + exp_ctrl |= RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP; + if (arg->mode == RKISP1_CIF_ISP_EXP_MEASURING_MODE_1) + exp_ctrl |= RKISP1_CIF_ISP_EXP_CTRL_MEASMODE_1; + rkisp1_write(params->rkisp1, exp_ctrl, RKISP1_CIF_ISP_EXP_CTRL); + + rkisp1_write(params->rkisp1, + arg->meas_window.h_offs, RKISP1_CIF_ISP_EXP_H_OFFSET); + rkisp1_write(params->rkisp1, + arg->meas_window.v_offs, RKISP1_CIF_ISP_EXP_V_OFFSET); + + block_hsize = arg->meas_window.h_size / + RKISP1_CIF_ISP_EXP_COLUMN_NUM - 1; + block_vsize = arg->meas_window.v_size / + RKISP1_CIF_ISP_EXP_ROW_NUM - 1; + + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_EXP_H_SIZE_SET(block_hsize), + RKISP1_CIF_ISP_EXP_H_SIZE); + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_EXP_V_SIZE_SET(block_vsize), + RKISP1_CIF_ISP_EXP_V_SIZE); +} + +static void rkisp1_cproc_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_cproc_config *arg) +{ + struct rkisp1_cif_isp_isp_other_cfg *cur_other_cfg = + ¶ms->cur_params.others; + struct rkisp1_cif_isp_ie_config *cur_ie_config = + &cur_other_cfg->ie_config; + u32 effect = cur_ie_config->effect; + u32 quantization = params->quantization; + + rkisp1_write(params->rkisp1, arg->contrast, RKISP1_CIF_C_PROC_CONTRAST); + rkisp1_write(params->rkisp1, arg->hue, RKISP1_CIF_C_PROC_HUE); + rkisp1_write(params->rkisp1, arg->sat, RKISP1_CIF_C_PROC_SATURATION); + rkisp1_write(params->rkisp1, arg->brightness, + RKISP1_CIF_C_PROC_BRIGHTNESS); + + if (quantization != V4L2_QUANTIZATION_FULL_RANGE || + effect != V4L2_COLORFX_NONE) { + rkisp1_param_clear_bits(params, RKISP1_CIF_C_PROC_CTRL, + RKISP1_CIF_C_PROC_YOUT_FULL | + RKISP1_CIF_C_PROC_YIN_FULL | + RKISP1_CIF_C_PROC_COUT_FULL); + } else { + rkisp1_param_set_bits(params, RKISP1_CIF_C_PROC_CTRL, + RKISP1_CIF_C_PROC_YOUT_FULL | + RKISP1_CIF_C_PROC_YIN_FULL | + RKISP1_CIF_C_PROC_COUT_FULL); + } +} + +static void rkisp1_hst_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_hst_config *arg) +{ + unsigned int block_hsize, block_vsize; + static const u32 hist_weight_regs[] = { + RKISP1_CIF_ISP_HIST_WEIGHT_00TO30, + RKISP1_CIF_ISP_HIST_WEIGHT_40TO21, + RKISP1_CIF_ISP_HIST_WEIGHT_31TO12, + RKISP1_CIF_ISP_HIST_WEIGHT_22TO03, + RKISP1_CIF_ISP_HIST_WEIGHT_13TO43, + RKISP1_CIF_ISP_HIST_WEIGHT_04TO34, + RKISP1_CIF_ISP_HIST_WEIGHT_44, + }; + const u8 *weight; + unsigned int i; + u32 hist_prop; + + /* avoid to override the old enable value */ + hist_prop = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_HIST_PROP); + hist_prop &= RKISP1_CIF_ISP_HIST_PROP_MODE_MASK; + hist_prop |= RKISP1_CIF_ISP_HIST_PREDIV_SET(arg->histogram_predivider); + rkisp1_write(params->rkisp1, hist_prop, RKISP1_CIF_ISP_HIST_PROP); + rkisp1_write(params->rkisp1, + arg->meas_window.h_offs, + RKISP1_CIF_ISP_HIST_H_OFFS); + rkisp1_write(params->rkisp1, + arg->meas_window.v_offs, + RKISP1_CIF_ISP_HIST_V_OFFS); + + block_hsize = arg->meas_window.h_size / + RKISP1_CIF_ISP_HIST_COLUMN_NUM - 1; + block_vsize = arg->meas_window.v_size / RKISP1_CIF_ISP_HIST_ROW_NUM - 1; + + rkisp1_write(params->rkisp1, block_hsize, RKISP1_CIF_ISP_HIST_H_SIZE); + rkisp1_write(params->rkisp1, block_vsize, RKISP1_CIF_ISP_HIST_V_SIZE); + + weight = arg->hist_weight; + for (i = 0; i < ARRAY_SIZE(hist_weight_regs); ++i, weight += 4) + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_HIST_WEIGHT_SET(weight[0], + weight[1], + weight[2], + weight[3]), + hist_weight_regs[i]); +} + +static void +rkisp1_hst_enable(struct rkisp1_params *params, + const struct rkisp1_cif_isp_hst_config *arg, bool en) +{ + if (en) { + u32 hist_prop = rkisp1_read(params->rkisp1, + RKISP1_CIF_ISP_HIST_PROP); + + hist_prop &= ~RKISP1_CIF_ISP_HIST_PROP_MODE_MASK; + hist_prop |= arg->mode; + rkisp1_param_set_bits(params, RKISP1_CIF_ISP_HIST_PROP, + hist_prop); + } else { + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_HIST_PROP, + RKISP1_CIF_ISP_HIST_PROP_MODE_MASK); + } +} + +static void rkisp1_afm_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_afc_config *arg) +{ + size_t num_of_win = min_t(size_t, ARRAY_SIZE(arg->afm_win), + arg->num_afm_win); + u32 afm_ctrl = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_AFM_CTRL); + unsigned int i; + + /* Switch off to configure. */ + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_AFM_CTRL, + RKISP1_CIF_ISP_AFM_ENA); + + for (i = 0; i < num_of_win; i++) { + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_AFM_WINDOW_X(arg->afm_win[i].h_offs) | + RKISP1_CIF_ISP_AFM_WINDOW_Y(arg->afm_win[i].v_offs), + RKISP1_CIF_ISP_AFM_LT_A + i * 8); + rkisp1_write(params->rkisp1, + RKISP1_CIF_ISP_AFM_WINDOW_X(arg->afm_win[i].h_size + + arg->afm_win[i].h_offs) | + RKISP1_CIF_ISP_AFM_WINDOW_Y(arg->afm_win[i].v_size + + arg->afm_win[i].v_offs), + RKISP1_CIF_ISP_AFM_RB_A + i * 8); + } + rkisp1_write(params->rkisp1, arg->thres, RKISP1_CIF_ISP_AFM_THRES); + rkisp1_write(params->rkisp1, arg->var_shift, + RKISP1_CIF_ISP_AFM_VAR_SHIFT); + /* restore afm status */ + rkisp1_write(params->rkisp1, afm_ctrl, RKISP1_CIF_ISP_AFM_CTRL); +} + +static void rkisp1_ie_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_ie_config *arg) +{ + u32 eff_ctrl; + + eff_ctrl = rkisp1_read(params->rkisp1, RKISP1_CIF_IMG_EFF_CTRL); + eff_ctrl &= ~RKISP1_CIF_IMG_EFF_CTRL_MODE_MASK; + + if (params->quantization == V4L2_QUANTIZATION_FULL_RANGE) + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_YCBCR_FULL; + + switch (arg->effect) { + case V4L2_COLORFX_SEPIA: + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_MODE_SEPIA; + break; + case V4L2_COLORFX_SET_CBCR: + rkisp1_write(params->rkisp1, arg->eff_tint, + RKISP1_CIF_IMG_EFF_TINT); + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_MODE_SEPIA; + break; + /* + * Color selection is similar to water color(AQUA): + * grayscale + selected color w threshold + */ + case V4L2_COLORFX_AQUA: + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_MODE_COLOR_SEL; + rkisp1_write(params->rkisp1, arg->color_sel, + RKISP1_CIF_IMG_EFF_COLOR_SEL); + break; + case V4L2_COLORFX_EMBOSS: + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_MODE_EMBOSS; + rkisp1_write(params->rkisp1, arg->eff_mat_1, + RKISP1_CIF_IMG_EFF_MAT_1); + rkisp1_write(params->rkisp1, arg->eff_mat_2, + RKISP1_CIF_IMG_EFF_MAT_2); + rkisp1_write(params->rkisp1, arg->eff_mat_3, + RKISP1_CIF_IMG_EFF_MAT_3); + break; + case V4L2_COLORFX_SKETCH: + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_MODE_SKETCH; + rkisp1_write(params->rkisp1, arg->eff_mat_3, + RKISP1_CIF_IMG_EFF_MAT_3); + rkisp1_write(params->rkisp1, arg->eff_mat_4, + RKISP1_CIF_IMG_EFF_MAT_4); + rkisp1_write(params->rkisp1, arg->eff_mat_5, + RKISP1_CIF_IMG_EFF_MAT_5); + break; + case V4L2_COLORFX_BW: + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_MODE_BLACKWHITE; + break; + case V4L2_COLORFX_NEGATIVE: + eff_ctrl |= RKISP1_CIF_IMG_EFF_CTRL_MODE_NEGATIVE; + break; + default: + break; + } + + rkisp1_write(params->rkisp1, eff_ctrl, RKISP1_CIF_IMG_EFF_CTRL); +} + +static void rkisp1_ie_enable(struct rkisp1_params *params, bool en) +{ + if (en) { + rkisp1_param_set_bits(params, RKISP1_CIF_ICCL, + RKISP1_CIF_ICCL_IE_CLK); + rkisp1_write(params->rkisp1, RKISP1_CIF_IMG_EFF_CTRL_ENABLE, + RKISP1_CIF_IMG_EFF_CTRL); + rkisp1_param_set_bits(params, RKISP1_CIF_IMG_EFF_CTRL, + RKISP1_CIF_IMG_EFF_CTRL_CFG_UPD); + } else { + rkisp1_param_clear_bits(params, RKISP1_CIF_IMG_EFF_CTRL, + RKISP1_CIF_IMG_EFF_CTRL_ENABLE); + rkisp1_param_clear_bits(params, RKISP1_CIF_ICCL, + RKISP1_CIF_ICCL_IE_CLK); + } +} + +static void rkisp1_csm_config(struct rkisp1_params *params, bool full_range) +{ + static const u16 full_range_coeff[] = { + 0x0026, 0x004b, 0x000f, + 0x01ea, 0x01d6, 0x0040, + 0x0040, 0x01ca, 0x01f6 + }; + static const u16 limited_range_coeff[] = { + 0x0021, 0x0040, 0x000d, + 0x01ed, 0x01db, 0x0038, + 0x0038, 0x01d1, 0x01f7, + }; + unsigned int i; + + if (full_range) { + for (i = 0; i < ARRAY_SIZE(full_range_coeff); i++) + rkisp1_write(params->rkisp1, full_range_coeff[i], + RKISP1_CIF_ISP_CC_COEFF_0 + i * 4); + + rkisp1_param_set_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_CSM_Y_FULL_ENA | + RKISP1_CIF_ISP_CTRL_ISP_CSM_C_FULL_ENA); + } else { + for (i = 0; i < ARRAY_SIZE(limited_range_coeff); i++) + rkisp1_write(params->rkisp1, limited_range_coeff[i], + RKISP1_CIF_ISP_CC_COEFF_0 + i * 4); + + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_CSM_Y_FULL_ENA | + RKISP1_CIF_ISP_CTRL_ISP_CSM_C_FULL_ENA); + } +} + +/* ISP De-noise Pre-Filter(DPF) function */ +static void rkisp1_dpf_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_dpf_config *arg) +{ + unsigned int isp_dpf_mode, spatial_coeff, i; + + switch (arg->gain.mode) { + case RKISP1_CIF_ISP_DPF_GAIN_USAGE_NF_GAINS: + isp_dpf_mode = RKISP1_CIF_ISP_DPF_MODE_USE_NF_GAIN | + RKISP1_CIF_ISP_DPF_MODE_AWB_GAIN_COMP; + break; + case RKISP1_CIF_ISP_DPF_GAIN_USAGE_LSC_GAINS: + isp_dpf_mode = RKISP1_CIF_ISP_DPF_MODE_LSC_GAIN_COMP; + break; + case RKISP1_CIF_ISP_DPF_GAIN_USAGE_NF_LSC_GAINS: + isp_dpf_mode = RKISP1_CIF_ISP_DPF_MODE_USE_NF_GAIN | + RKISP1_CIF_ISP_DPF_MODE_AWB_GAIN_COMP | + RKISP1_CIF_ISP_DPF_MODE_LSC_GAIN_COMP; + break; + case RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_GAINS: + isp_dpf_mode = RKISP1_CIF_ISP_DPF_MODE_AWB_GAIN_COMP; + break; + case RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_LSC_GAINS: + isp_dpf_mode = RKISP1_CIF_ISP_DPF_MODE_LSC_GAIN_COMP | + RKISP1_CIF_ISP_DPF_MODE_AWB_GAIN_COMP; + break; + case RKISP1_CIF_ISP_DPF_GAIN_USAGE_DISABLED: + default: + isp_dpf_mode = 0; + break; + } + + if (arg->nll.scale_mode == RKISP1_CIF_ISP_NLL_SCALE_LOGARITHMIC) + isp_dpf_mode |= RKISP1_CIF_ISP_DPF_MODE_NLL_SEGMENTATION; + if (arg->rb_flt.fltsize == RKISP1_CIF_ISP_DPF_RB_FILTERSIZE_9x9) + isp_dpf_mode |= RKISP1_CIF_ISP_DPF_MODE_RB_FLTSIZE_9x9; + if (!arg->rb_flt.r_enable) + isp_dpf_mode |= RKISP1_CIF_ISP_DPF_MODE_R_FLT_DIS; + if (!arg->rb_flt.b_enable) + isp_dpf_mode |= RKISP1_CIF_ISP_DPF_MODE_B_FLT_DIS; + if (!arg->g_flt.gb_enable) + isp_dpf_mode |= RKISP1_CIF_ISP_DPF_MODE_GB_FLT_DIS; + if (!arg->g_flt.gr_enable) + isp_dpf_mode |= RKISP1_CIF_ISP_DPF_MODE_GR_FLT_DIS; + + rkisp1_param_set_bits(params, RKISP1_CIF_ISP_DPF_MODE, + isp_dpf_mode); + rkisp1_write(params->rkisp1, arg->gain.nf_b_gain, + RKISP1_CIF_ISP_DPF_NF_GAIN_B); + rkisp1_write(params->rkisp1, arg->gain.nf_r_gain, + RKISP1_CIF_ISP_DPF_NF_GAIN_R); + rkisp1_write(params->rkisp1, arg->gain.nf_gb_gain, + RKISP1_CIF_ISP_DPF_NF_GAIN_GB); + rkisp1_write(params->rkisp1, arg->gain.nf_gr_gain, + RKISP1_CIF_ISP_DPF_NF_GAIN_GR); + + for (i = 0; i < RKISP1_CIF_ISP_DPF_MAX_NLF_COEFFS; i++) { + rkisp1_write(params->rkisp1, arg->nll.coeff[i], + RKISP1_CIF_ISP_DPF_NULL_COEFF_0 + i * 4); + } + + spatial_coeff = arg->g_flt.spatial_coeff[0] | + (arg->g_flt.spatial_coeff[1] << 8) | + (arg->g_flt.spatial_coeff[2] << 16) | + (arg->g_flt.spatial_coeff[3] << 24); + rkisp1_write(params->rkisp1, spatial_coeff, + RKISP1_CIF_ISP_DPF_S_WEIGHT_G_1_4); + + spatial_coeff = arg->g_flt.spatial_coeff[4] | + (arg->g_flt.spatial_coeff[5] << 8); + rkisp1_write(params->rkisp1, spatial_coeff, + RKISP1_CIF_ISP_DPF_S_WEIGHT_G_5_6); + + spatial_coeff = arg->rb_flt.spatial_coeff[0] | + (arg->rb_flt.spatial_coeff[1] << 8) | + (arg->rb_flt.spatial_coeff[2] << 16) | + (arg->rb_flt.spatial_coeff[3] << 24); + rkisp1_write(params->rkisp1, spatial_coeff, + RKISP1_CIF_ISP_DPF_S_WEIGHT_RB_1_4); + + spatial_coeff = arg->rb_flt.spatial_coeff[4] | + (arg->rb_flt.spatial_coeff[5] << 8); + rkisp1_write(params->rkisp1, spatial_coeff, + RKISP1_CIF_ISP_DPF_S_WEIGHT_RB_5_6); +} + +static void +rkisp1_dpf_strength_config(struct rkisp1_params *params, + const struct rkisp1_cif_isp_dpf_strength_config *arg) +{ + rkisp1_write(params->rkisp1, arg->b, RKISP1_CIF_ISP_DPF_STRENGTH_B); + rkisp1_write(params->rkisp1, arg->g, RKISP1_CIF_ISP_DPF_STRENGTH_G); + rkisp1_write(params->rkisp1, arg->r, RKISP1_CIF_ISP_DPF_STRENGTH_R); +} + +static void +rkisp1_isp_isr_other_config(struct rkisp1_params *params, + const struct rkisp1_params_cfg *new_params) +{ + unsigned int module_en_update, module_cfg_update, module_ens; + + module_en_update = new_params->module_en_update; + module_cfg_update = new_params->module_cfg_update; + module_ens = new_params->module_ens; + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_DPCC) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_DPCC)) { + /*update dpc config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_DPCC)) + rkisp1_dpcc_config(params, + &new_params->others.dpcc_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_DPCC) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_DPCC)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_DPCC_MODE, + RKISP1_CIF_ISP_DPCC_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_DPCC_MODE, + RKISP1_CIF_ISP_DPCC_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_BLS) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_BLS)) { + /* update bls config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_BLS)) + rkisp1_bls_config(params, + &new_params->others.bls_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_BLS) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_BLS)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_BLS_CTRL, + RKISP1_CIF_ISP_BLS_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_BLS_CTRL, + RKISP1_CIF_ISP_BLS_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_SDG) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_SDG)) { + /* update sdg config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_SDG)) + rkisp1_sdg_config(params, + &new_params->others.sdg_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_SDG) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_SDG)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_GAMMA_IN_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_GAMMA_IN_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_LSC) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_LSC)) { + /* update lsc config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_LSC)) + rkisp1_lsc_config(params, + &new_params->others.lsc_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_LSC) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_LSC)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_LSC_CTRL, + RKISP1_CIF_ISP_LSC_CTRL_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_LSC_CTRL, + RKISP1_CIF_ISP_LSC_CTRL_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_AWB_GAIN) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_AWB_GAIN)) { + /* update awb gains */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_AWB_GAIN)) + rkisp1_awb_gain_config(params, + &new_params->others.awb_gain_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_AWB_GAIN) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_AWB_GAIN)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_AWB_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_AWB_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_BDM) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_BDM)) { + /* update bdm config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_BDM)) + rkisp1_bdm_config(params, + &new_params->others.bdm_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_BDM) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_BDM)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_DEMOSAIC, + RKISP1_CIF_ISP_DEMOSAIC_BYPASS); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_DEMOSAIC, + RKISP1_CIF_ISP_DEMOSAIC_BYPASS); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_FLT) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_FLT)) { + /* update filter config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_FLT)) + rkisp1_flt_config(params, + &new_params->others.flt_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_FLT) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_FLT)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_FILT_MODE, + RKISP1_CIF_ISP_FLT_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_FILT_MODE, + RKISP1_CIF_ISP_FLT_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_CTK) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_CTK)) { + /* update ctk config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_CTK)) + rkisp1_ctk_config(params, + &new_params->others.ctk_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_CTK) + rkisp1_ctk_enable(params, + !!(module_ens & RKISP1_CIF_ISP_MODULE_CTK)); + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_GOC) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_GOC)) { + /* update goc config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_GOC)) + rkisp1_goc_config(params, + &new_params->others.goc_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_GOC) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_GOC)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_GAMMA_OUT_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_GAMMA_OUT_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_CPROC) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_CPROC)) { + /* update cproc config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_CPROC)) { + rkisp1_cproc_config(params, + &new_params->others.cproc_config); + } + + if (module_en_update & RKISP1_CIF_ISP_MODULE_CPROC) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_CPROC)) + rkisp1_param_set_bits(params, + RKISP1_CIF_C_PROC_CTRL, + RKISP1_CIF_C_PROC_CTR_ENABLE); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_C_PROC_CTRL, + RKISP1_CIF_C_PROC_CTR_ENABLE); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_IE) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_IE)) { + /* update ie config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_IE)) + rkisp1_ie_config(params, + &new_params->others.ie_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_IE) + rkisp1_ie_enable(params, + !!(module_ens & RKISP1_CIF_ISP_MODULE_IE)); + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_DPF) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_DPF)) { + /* update dpf config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_DPF)) + rkisp1_dpf_config(params, + &new_params->others.dpf_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_DPF) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_DPF)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_DPF_MODE, + RKISP1_CIF_ISP_DPF_MODE_EN); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_DPF_MODE, + RKISP1_CIF_ISP_DPF_MODE_EN); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_DPF_STRENGTH) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_DPF_STRENGTH)) { + /* update dpf strength config */ + rkisp1_dpf_strength_config(params, + &new_params->others.dpf_strength_config); + } +} + +static void rkisp1_isp_isr_meas_config(struct rkisp1_params *params, + struct rkisp1_params_cfg *new_params) +{ + unsigned int module_en_update, module_cfg_update, module_ens; + + module_en_update = new_params->module_en_update; + module_cfg_update = new_params->module_cfg_update; + module_ens = new_params->module_ens; + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_AWB) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_AWB)) { + /* update awb config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_AWB)) + rkisp1_awb_meas_config(params, + &new_params->meas.awb_meas_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_AWB) + rkisp1_awb_meas_enable(params, + &new_params->meas.awb_meas_config, + !!(module_ens & RKISP1_CIF_ISP_MODULE_AWB)); + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_AFC) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_AFC)) { + /* update afc config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_AFC)) + rkisp1_afm_config(params, + &new_params->meas.afc_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_AFC) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_AFC)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_AFM_CTRL, + RKISP1_CIF_ISP_AFM_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_AFM_CTRL, + RKISP1_CIF_ISP_AFM_ENA); + } + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_HST) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_HST)) { + /* update hst config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_HST)) + rkisp1_hst_config(params, + &new_params->meas.hst_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_HST) + rkisp1_hst_enable(params, + &new_params->meas.hst_config, + !!(module_ens & RKISP1_CIF_ISP_MODULE_HST)); + } + + if ((module_en_update & RKISP1_CIF_ISP_MODULE_AEC) || + (module_cfg_update & RKISP1_CIF_ISP_MODULE_AEC)) { + /* update aec config */ + if ((module_cfg_update & RKISP1_CIF_ISP_MODULE_AEC)) + rkisp1_aec_config(params, + &new_params->meas.aec_config); + + if (module_en_update & RKISP1_CIF_ISP_MODULE_AEC) { + if (!!(module_ens & RKISP1_CIF_ISP_MODULE_AEC)) + rkisp1_param_set_bits(params, + RKISP1_CIF_ISP_EXP_CTRL, + RKISP1_CIF_ISP_EXP_ENA); + else + rkisp1_param_clear_bits(params, + RKISP1_CIF_ISP_EXP_CTRL, + RKISP1_CIF_ISP_EXP_ENA); + } + } +} + +void rkisp1_params_isr(struct rkisp1_device *rkisp1, u32 isp_mis) +{ + unsigned int frame_sequence = atomic_read(&rkisp1->isp.frame_sequence); + struct rkisp1_params *params = &rkisp1->params; + struct rkisp1_params_cfg *new_params; + struct rkisp1_buffer *cur_buf = NULL; + + spin_lock(¶ms->config_lock); + if (!params->is_streaming) { + spin_unlock(¶ms->config_lock); + return; + } + + /* get one empty buffer */ + if (!list_empty(¶ms->params)) + cur_buf = list_first_entry(¶ms->params, + struct rkisp1_buffer, queue); + spin_unlock(¶ms->config_lock); + + if (!cur_buf) + return; + + new_params = (struct rkisp1_params_cfg *)(cur_buf->vaddr[0]); + + if (isp_mis & RKISP1_CIF_ISP_FRAME) { + u32 isp_ctrl; + + rkisp1_isp_isr_other_config(params, new_params); + rkisp1_isp_isr_meas_config(params, new_params); + + /* update shadow register immediately */ + isp_ctrl = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_CTRL); + isp_ctrl |= RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD; + rkisp1_write(params->rkisp1, isp_ctrl, RKISP1_CIF_ISP_CTRL); + + spin_lock(¶ms->config_lock); + list_del(&cur_buf->queue); + spin_unlock(¶ms->config_lock); + + cur_buf->vb.sequence = frame_sequence; + vb2_buffer_done(&cur_buf->vb.vb2_buf, VB2_BUF_STATE_DONE); + } +} + +static const struct rkisp1_cif_isp_awb_meas_config rkisp1_awb_params_default_config = { + { + 0, 0, RKISP1_DEFAULT_WIDTH, RKISP1_DEFAULT_HEIGHT + }, + RKISP1_CIF_ISP_AWB_MODE_YCBCR, 200, 30, 20, 20, 0, 128, 128 +}; + +static const struct rkisp1_cif_isp_aec_config rkisp1_aec_params_default_config = { + RKISP1_CIF_ISP_EXP_MEASURING_MODE_0, + RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0, + { + RKISP1_DEFAULT_WIDTH >> 2, RKISP1_DEFAULT_HEIGHT >> 2, + RKISP1_DEFAULT_WIDTH >> 1, RKISP1_DEFAULT_HEIGHT >> 1 + } +}; + +static const struct rkisp1_cif_isp_hst_config rkisp1_hst_params_default_config = { + RKISP1_CIF_ISP_HISTOGRAM_MODE_RGB_COMBINED, + 3, + { + RKISP1_DEFAULT_WIDTH >> 2, RKISP1_DEFAULT_HEIGHT >> 2, + RKISP1_DEFAULT_WIDTH >> 1, RKISP1_DEFAULT_HEIGHT >> 1 + }, + { + 0, /* To be filled in with 0x01 at runtime. */ + } +}; + +static const struct rkisp1_cif_isp_afc_config rkisp1_afc_params_default_config = { + 1, + { + { + 300, 225, 200, 150 + } + }, + 4, + 14 +}; + +static void rkisp1_params_config_parameter(struct rkisp1_params *params) +{ + struct rkisp1_cif_isp_hst_config hst = rkisp1_hst_params_default_config; + + spin_lock(¶ms->config_lock); + + rkisp1_awb_meas_config(params, &rkisp1_awb_params_default_config); + rkisp1_awb_meas_enable(params, &rkisp1_awb_params_default_config, + true); + + rkisp1_aec_config(params, &rkisp1_aec_params_default_config); + rkisp1_param_set_bits(params, RKISP1_CIF_ISP_EXP_CTRL, + RKISP1_CIF_ISP_EXP_ENA); + + rkisp1_afm_config(params, &rkisp1_afc_params_default_config); + rkisp1_param_set_bits(params, RKISP1_CIF_ISP_AFM_CTRL, + RKISP1_CIF_ISP_AFM_ENA); + + memset(hst.hist_weight, 0x01, sizeof(hst.hist_weight)); + rkisp1_hst_config(params, &hst); + rkisp1_param_set_bits(params, RKISP1_CIF_ISP_HIST_PROP, + ~RKISP1_CIF_ISP_HIST_PROP_MODE_MASK | + rkisp1_hst_params_default_config.mode); + + /* set the range */ + if (params->quantization == V4L2_QUANTIZATION_FULL_RANGE) + rkisp1_csm_config(params, true); + else + rkisp1_csm_config(params, false); + + /* override the default things */ + rkisp1_isp_isr_other_config(params, ¶ms->cur_params); + rkisp1_isp_isr_meas_config(params, ¶ms->cur_params); + + spin_unlock(¶ms->config_lock); +} + +/* Not called when the camera active, thus not isr protection. */ +void rkisp1_params_configure(struct rkisp1_params *params, + enum rkisp1_fmt_raw_pat_type bayer_pat, + enum v4l2_quantization quantization) +{ + params->quantization = quantization; + params->raw_type = bayer_pat; + rkisp1_params_config_parameter(params); +} + +/* Not called when the camera active, thus not isr protection. */ +void rkisp1_params_disable(struct rkisp1_params *params) +{ + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_DPCC_MODE, + RKISP1_CIF_ISP_DPCC_ENA); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_LSC_CTRL, + RKISP1_CIF_ISP_LSC_CTRL_ENA); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_BLS_CTRL, + RKISP1_CIF_ISP_BLS_ENA); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_GAMMA_IN_ENA); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_GAMMA_OUT_ENA); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_DEMOSAIC, + RKISP1_CIF_ISP_DEMOSAIC_BYPASS); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_FILT_MODE, + RKISP1_CIF_ISP_FLT_ENA); + rkisp1_awb_meas_enable(params, NULL, false); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CTRL, + RKISP1_CIF_ISP_CTRL_ISP_AWB_ENA); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_EXP_CTRL, + RKISP1_CIF_ISP_EXP_ENA); + rkisp1_ctk_enable(params, false); + rkisp1_param_clear_bits(params, RKISP1_CIF_C_PROC_CTRL, + RKISP1_CIF_C_PROC_CTR_ENABLE); + rkisp1_hst_enable(params, NULL, false); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_AFM_CTRL, + RKISP1_CIF_ISP_AFM_ENA); + rkisp1_ie_enable(params, false); + rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_DPF_MODE, + RKISP1_CIF_ISP_DPF_MODE_EN); +} + +static int rkisp1_params_enum_fmt_meta_out(struct file *file, void *priv, + struct v4l2_fmtdesc *f) +{ + struct video_device *video = video_devdata(file); + struct rkisp1_params *params = video_get_drvdata(video); + + if (f->index > 0 || f->type != video->queue->type) + return -EINVAL; + + f->pixelformat = params->vdev_fmt.fmt.meta.dataformat; + + return 0; +} + +static int rkisp1_params_g_fmt_meta_out(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct video_device *video = video_devdata(file); + struct rkisp1_params *params = video_get_drvdata(video); + struct v4l2_meta_format *meta = &f->fmt.meta; + + if (f->type != video->queue->type) + return -EINVAL; + + memset(meta, 0, sizeof(*meta)); + meta->dataformat = params->vdev_fmt.fmt.meta.dataformat; + meta->buffersize = params->vdev_fmt.fmt.meta.buffersize; + + return 0; +} + +static int rkisp1_params_querycap(struct file *file, + void *priv, struct v4l2_capability *cap) +{ + struct video_device *vdev = video_devdata(file); + + strscpy(cap->driver, RKISP1_DRIVER_NAME, sizeof(cap->driver)); + strscpy(cap->card, vdev->name, sizeof(cap->card)); + strscpy(cap->bus_info, RKISP1_BUS_INFO, sizeof(cap->bus_info)); + + return 0; +} + +/* ISP params video device IOCTLs */ +static const struct v4l2_ioctl_ops rkisp1_params_ioctl = { + .vidioc_reqbufs = vb2_ioctl_reqbufs, + .vidioc_querybuf = vb2_ioctl_querybuf, + .vidioc_create_bufs = vb2_ioctl_create_bufs, + .vidioc_qbuf = vb2_ioctl_qbuf, + .vidioc_dqbuf = vb2_ioctl_dqbuf, + .vidioc_prepare_buf = vb2_ioctl_prepare_buf, + .vidioc_expbuf = vb2_ioctl_expbuf, + .vidioc_streamon = vb2_ioctl_streamon, + .vidioc_streamoff = vb2_ioctl_streamoff, + .vidioc_enum_fmt_meta_out = rkisp1_params_enum_fmt_meta_out, + .vidioc_g_fmt_meta_out = rkisp1_params_g_fmt_meta_out, + .vidioc_s_fmt_meta_out = rkisp1_params_g_fmt_meta_out, + .vidioc_try_fmt_meta_out = rkisp1_params_g_fmt_meta_out, + .vidioc_querycap = rkisp1_params_querycap, + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +static int rkisp1_params_vb2_queue_setup(struct vb2_queue *vq, + unsigned int *num_buffers, + unsigned int *num_planes, + unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct rkisp1_params *params = vq->drv_priv; + + *num_buffers = clamp_t(u32, *num_buffers, + RKISP1_ISP_PARAMS_REQ_BUFS_MIN, + RKISP1_ISP_PARAMS_REQ_BUFS_MAX); + + *num_planes = 1; + + sizes[0] = sizeof(struct rkisp1_params_cfg); + + INIT_LIST_HEAD(¶ms->params); + params->is_first_params = true; + + return 0; +} + +static void rkisp1_params_vb2_buf_queue(struct vb2_buffer *vb) +{ + struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); + struct rkisp1_buffer *params_buf = + container_of(vbuf, struct rkisp1_buffer, vb); + struct vb2_queue *vq = vb->vb2_queue; + struct rkisp1_params *params = vq->drv_priv; + struct rkisp1_params_cfg *new_params; + unsigned long flags; + unsigned int frame_sequence = + atomic_read(¶ms->rkisp1->isp.frame_sequence); + + if (params->is_first_params) { + new_params = (struct rkisp1_params_cfg *) + (vb2_plane_vaddr(vb, 0)); + vbuf->sequence = frame_sequence; + vb2_buffer_done(¶ms_buf->vb.vb2_buf, VB2_BUF_STATE_DONE); + params->is_first_params = false; + params->cur_params = *new_params; + return; + } + + params_buf->vaddr[0] = vb2_plane_vaddr(vb, 0); + spin_lock_irqsave(¶ms->config_lock, flags); + list_add_tail(¶ms_buf->queue, ¶ms->params); + spin_unlock_irqrestore(¶ms->config_lock, flags); +} + +static int rkisp1_params_vb2_buf_prepare(struct vb2_buffer *vb) +{ + if (vb2_plane_size(vb, 0) < sizeof(struct rkisp1_params_cfg)) + return -EINVAL; + + vb2_set_plane_payload(vb, 0, sizeof(struct rkisp1_params_cfg)); + + return 0; +} + +static void rkisp1_params_vb2_stop_streaming(struct vb2_queue *vq) +{ + struct rkisp1_params *params = vq->drv_priv; + struct rkisp1_buffer *buf; + unsigned long flags; + unsigned int i; + + /* stop params input firstly */ + spin_lock_irqsave(¶ms->config_lock, flags); + params->is_streaming = false; + spin_unlock_irqrestore(¶ms->config_lock, flags); + + for (i = 0; i < RKISP1_ISP_PARAMS_REQ_BUFS_MAX; i++) { + spin_lock_irqsave(¶ms->config_lock, flags); + if (!list_empty(¶ms->params)) { + buf = list_first_entry(¶ms->params, + struct rkisp1_buffer, queue); + list_del(&buf->queue); + spin_unlock_irqrestore(¶ms->config_lock, + flags); + } else { + spin_unlock_irqrestore(¶ms->config_lock, + flags); + break; + } + + if (buf) + vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR); + buf = NULL; + } +} + +static int +rkisp1_params_vb2_start_streaming(struct vb2_queue *queue, unsigned int count) +{ + struct rkisp1_params *params = queue->drv_priv; + unsigned long flags; + + spin_lock_irqsave(¶ms->config_lock, flags); + params->is_streaming = true; + spin_unlock_irqrestore(¶ms->config_lock, flags); + + return 0; +} + +static struct vb2_ops rkisp1_params_vb2_ops = { + .queue_setup = rkisp1_params_vb2_queue_setup, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, + .buf_queue = rkisp1_params_vb2_buf_queue, + .buf_prepare = rkisp1_params_vb2_buf_prepare, + .start_streaming = rkisp1_params_vb2_start_streaming, + .stop_streaming = rkisp1_params_vb2_stop_streaming, + +}; + +static struct v4l2_file_operations rkisp1_params_fops = { + .mmap = vb2_fop_mmap, + .unlocked_ioctl = video_ioctl2, + .poll = vb2_fop_poll, + .open = v4l2_fh_open, + .release = vb2_fop_release +}; + +static int rkisp1_params_init_vb2_queue(struct vb2_queue *q, + struct rkisp1_params *params) +{ + struct rkisp1_vdev_node *node; + + node = container_of(q, struct rkisp1_vdev_node, buf_queue); + + q->type = V4L2_BUF_TYPE_META_OUTPUT; + q->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF; + q->drv_priv = params; + q->ops = &rkisp1_params_vb2_ops; + q->mem_ops = &vb2_vmalloc_memops; + q->buf_struct_size = sizeof(struct rkisp1_buffer); + q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + q->lock = &node->vlock; + + return vb2_queue_init(q); +} + +static void rkisp1_init_params(struct rkisp1_params *params) +{ + params->vdev_fmt.fmt.meta.dataformat = + V4L2_META_FMT_RK_ISP1_PARAMS; + params->vdev_fmt.fmt.meta.buffersize = + sizeof(struct rkisp1_params_cfg); +} + +int rkisp1_params_register(struct rkisp1_params *params, + struct v4l2_device *v4l2_dev, + struct rkisp1_device *rkisp1) +{ + struct rkisp1_vdev_node *node = ¶ms->vnode; + struct video_device *vdev = &node->vdev; + int ret; + + params->rkisp1 = rkisp1; + mutex_init(&node->vlock); + spin_lock_init(¶ms->config_lock); + + strscpy(vdev->name, RKISP1_PARAMS_DEV_NAME, sizeof(vdev->name)); + + video_set_drvdata(vdev, params); + vdev->ioctl_ops = &rkisp1_params_ioctl; + vdev->fops = &rkisp1_params_fops; + vdev->release = video_device_release_empty; + /* + * Provide a mutex to v4l2 core. It will be used + * to protect all fops and v4l2 ioctls. + */ + vdev->lock = &node->vlock; + vdev->v4l2_dev = v4l2_dev; + vdev->queue = &node->buf_queue; + vdev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_META_OUTPUT; + vdev->vfl_dir = VFL_DIR_TX; + rkisp1_params_init_vb2_queue(vdev->queue, params); + rkisp1_init_params(params); + video_set_drvdata(vdev, params); + + node->pad.flags = MEDIA_PAD_FL_SOURCE; + ret = media_entity_pads_init(&vdev->entity, 1, &node->pad); + if (ret) + goto err_release_queue; + ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1); + if (ret) { + dev_err(&vdev->dev, + "failed to register %s, ret=%d\n", vdev->name, ret); + goto err_cleanup_media_entity; + } + return 0; +err_cleanup_media_entity: + media_entity_cleanup(&vdev->entity); +err_release_queue: + vb2_queue_release(vdev->queue); + return ret; +} + +void rkisp1_params_unregister(struct rkisp1_params *params) +{ + struct rkisp1_vdev_node *node = ¶ms->vnode; + struct video_device *vdev = &node->vdev; + + video_unregister_device(vdev); + media_entity_cleanup(&vdev->entity); + vb2_queue_release(vdev->queue); +} diff --git a/drivers/staging/media/rkisp1/rkisp1-regs.h b/drivers/staging/media/rkisp1/rkisp1-regs.h new file mode 100644 index 000000000000..46018f435b6f --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-regs.h @@ -0,0 +1,1264 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Rockchip ISP1 Driver - Registers header + * + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#ifndef _RKISP1_REGS_H +#define _RKISP1_REGS_H + +/* ISP_CTRL */ +#define RKISP1_CIF_ISP_CTRL_ISP_ENABLE BIT(0) +#define RKISP1_CIF_ISP_CTRL_ISP_MODE_RAW_PICT (0 << 1) +#define RKISP1_CIF_ISP_CTRL_ISP_MODE_ITU656 BIT(1) +#define RKISP1_CIF_ISP_CTRL_ISP_MODE_ITU601 (2 << 1) +#define RKISP1_CIF_ISP_CTRL_ISP_MODE_BAYER_ITU601 (3 << 1) +#define RKISP1_CIF_ISP_CTRL_ISP_MODE_DATA_MODE (4 << 1) +#define RKISP1_CIF_ISP_CTRL_ISP_MODE_BAYER_ITU656 (5 << 1) +#define RKISP1_CIF_ISP_CTRL_ISP_MODE_RAW_PICT_ITU656 (6 << 1) +#define RKISP1_CIF_ISP_CTRL_ISP_INFORM_ENABLE BIT(4) +#define RKISP1_CIF_ISP_CTRL_ISP_GAMMA_IN_ENA BIT(6) +#define RKISP1_CIF_ISP_CTRL_ISP_AWB_ENA BIT(7) +#define RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD_PERMANENT BIT(8) +#define RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD BIT(9) +#define RKISP1_CIF_ISP_CTRL_ISP_GEN_CFG_UPD BIT(10) +#define RKISP1_CIF_ISP_CTRL_ISP_GAMMA_OUT_ENA BIT(11) +#define RKISP1_CIF_ISP_CTRL_ISP_FLASH_MODE_ENA BIT(12) +#define RKISP1_CIF_ISP_CTRL_ISP_CSM_Y_FULL_ENA BIT(13) +#define RKISP1_CIF_ISP_CTRL_ISP_CSM_C_FULL_ENA BIT(14) + +/* ISP_ACQ_PROP */ +#define RKISP1_CIF_ISP_ACQ_PROP_POS_EDGE BIT(0) +#define RKISP1_CIF_ISP_ACQ_PROP_HSYNC_LOW BIT(1) +#define RKISP1_CIF_ISP_ACQ_PROP_VSYNC_LOW BIT(2) +#define RKISP1_CIF_ISP_ACQ_PROP_BAYER_PAT_RGGB (0 << 3) +#define RKISP1_CIF_ISP_ACQ_PROP_BAYER_PAT_GRBG BIT(3) +#define RKISP1_CIF_ISP_ACQ_PROP_BAYER_PAT_GBRG (2 << 3) +#define RKISP1_CIF_ISP_ACQ_PROP_BAYER_PAT_BGGR (3 << 3) +#define RKISP1_CIF_ISP_ACQ_PROP_BAYER_PAT(pat) ((pat) << 3) +#define RKISP1_CIF_ISP_ACQ_PROP_YCBYCR (0 << 7) +#define RKISP1_CIF_ISP_ACQ_PROP_YCRYCB BIT(7) +#define RKISP1_CIF_ISP_ACQ_PROP_CBYCRY (2 << 7) +#define RKISP1_CIF_ISP_ACQ_PROP_CRYCBY (3 << 7) +#define RKISP1_CIF_ISP_ACQ_PROP_FIELD_SEL_ALL (0 << 9) +#define RKISP1_CIF_ISP_ACQ_PROP_FIELD_SEL_EVEN BIT(9) +#define RKISP1_CIF_ISP_ACQ_PROP_FIELD_SEL_ODD (2 << 9) +#define RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_12B (0 << 12) +#define RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_10B_ZERO BIT(12) +#define RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_10B_MSB (2 << 12) +#define RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_8B_ZERO (3 << 12) +#define RKISP1_CIF_ISP_ACQ_PROP_IN_SEL_8B_MSB (4 << 12) + +/* VI_DPCL */ +#define RKISP1_CIF_VI_DPCL_DMA_JPEG (0 << 0) +#define RKISP1_CIF_VI_DPCL_MP_MUX_MRSZ_MI BIT(0) +#define RKISP1_CIF_VI_DPCL_MP_MUX_MRSZ_JPEG (2 << 0) +#define RKISP1_CIF_VI_DPCL_CHAN_MODE_MP BIT(2) +#define RKISP1_CIF_VI_DPCL_CHAN_MODE_SP (2 << 2) +#define RKISP1_CIF_VI_DPCL_CHAN_MODE_MPSP (3 << 2) +#define RKISP1_CIF_VI_DPCL_DMA_SW_SPMUX (0 << 4) +#define RKISP1_CIF_VI_DPCL_DMA_SW_SI BIT(4) +#define RKISP1_CIF_VI_DPCL_DMA_SW_IE (2 << 4) +#define RKISP1_CIF_VI_DPCL_DMA_SW_JPEG (3 << 4) +#define RKISP1_CIF_VI_DPCL_DMA_SW_ISP (4 << 4) +#define RKISP1_CIF_VI_DPCL_IF_SEL_PARALLEL (0 << 8) +#define RKISP1_CIF_VI_DPCL_IF_SEL_SMIA BIT(8) +#define RKISP1_CIF_VI_DPCL_IF_SEL_MIPI (2 << 8) +#define RKISP1_CIF_VI_DPCL_DMA_IE_MUX_DMA BIT(10) +#define RKISP1_CIF_VI_DPCL_DMA_SP_MUX_DMA BIT(11) + +/* ISP_IMSC - ISP_MIS - ISP_RIS - ISP_ICR - ISP_ISR */ +#define RKISP1_CIF_ISP_OFF BIT(0) +#define RKISP1_CIF_ISP_FRAME BIT(1) +#define RKISP1_CIF_ISP_DATA_LOSS BIT(2) +#define RKISP1_CIF_ISP_PIC_SIZE_ERROR BIT(3) +#define RKISP1_CIF_ISP_AWB_DONE BIT(4) +#define RKISP1_CIF_ISP_FRAME_IN BIT(5) +#define RKISP1_CIF_ISP_V_START BIT(6) +#define RKISP1_CIF_ISP_H_START BIT(7) +#define RKISP1_CIF_ISP_FLASH_ON BIT(8) +#define RKISP1_CIF_ISP_FLASH_OFF BIT(9) +#define RKISP1_CIF_ISP_SHUTTER_ON BIT(10) +#define RKISP1_CIF_ISP_SHUTTER_OFF BIT(11) +#define RKISP1_CIF_ISP_AFM_SUM_OF BIT(12) +#define RKISP1_CIF_ISP_AFM_LUM_OF BIT(13) +#define RKISP1_CIF_ISP_AFM_FIN BIT(14) +#define RKISP1_CIF_ISP_HIST_MEASURE_RDY BIT(15) +#define RKISP1_CIF_ISP_FLASH_CAP BIT(17) +#define RKISP1_CIF_ISP_EXP_END BIT(18) +#define RKISP1_CIF_ISP_VSM_END BIT(19) + +/* ISP_ERR */ +#define RKISP1_CIF_ISP_ERR_INFORM_SIZE BIT(0) +#define RKISP1_CIF_ISP_ERR_IS_SIZE BIT(1) +#define RKISP1_CIF_ISP_ERR_OUTFORM_SIZE BIT(2) + +/* MI_CTRL */ +#define RKISP1_CIF_MI_CTRL_MP_ENABLE BIT(0) +#define RKISP1_CIF_MI_CTRL_SP_ENABLE (2 << 0) +#define RKISP1_CIF_MI_CTRL_JPEG_ENABLE (4 << 0) +#define RKISP1_CIF_MI_CTRL_RAW_ENABLE (8 << 0) +#define RKISP1_CIF_MI_CTRL_HFLIP BIT(4) +#define RKISP1_CIF_MI_CTRL_VFLIP BIT(5) +#define RKISP1_CIF_MI_CTRL_ROT BIT(6) +#define RKISP1_CIF_MI_BYTE_SWAP BIT(7) +#define RKISP1_CIF_MI_SP_Y_FULL_YUV2RGB BIT(8) +#define RKISP1_CIF_MI_SP_CBCR_FULL_YUV2RGB BIT(9) +#define RKISP1_CIF_MI_SP_422NONCOSITEED BIT(10) +#define RKISP1_CIF_MI_MP_PINGPONG_ENABEL BIT(11) +#define RKISP1_CIF_MI_SP_PINGPONG_ENABEL BIT(12) +#define RKISP1_CIF_MI_MP_AUTOUPDATE_ENABLE BIT(13) +#define RKISP1_CIF_MI_SP_AUTOUPDATE_ENABLE BIT(14) +#define RKISP1_CIF_MI_LAST_PIXEL_SIG_ENABLE BIT(15) +#define RKISP1_CIF_MI_CTRL_BURST_LEN_LUM_16 (0 << 16) +#define RKISP1_CIF_MI_CTRL_BURST_LEN_LUM_32 BIT(16) +#define RKISP1_CIF_MI_CTRL_BURST_LEN_LUM_64 (2 << 16) +#define RKISP1_CIF_MI_CTRL_BURST_LEN_CHROM_16 (0 << 18) +#define RKISP1_CIF_MI_CTRL_BURST_LEN_CHROM_32 BIT(18) +#define RKISP1_CIF_MI_CTRL_BURST_LEN_CHROM_64 (2 << 18) +#define RKISP1_CIF_MI_CTRL_INIT_BASE_EN BIT(20) +#define RKISP1_CIF_MI_CTRL_INIT_OFFSET_EN BIT(21) +#define RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8 (0 << 22) +#define RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA BIT(22) +#define RKISP1_MI_CTRL_MP_WRITE_YUVINT (2 << 22) +#define RKISP1_MI_CTRL_MP_WRITE_RAW12 (2 << 22) +#define RKISP1_MI_CTRL_SP_WRITE_PLA (0 << 24) +#define RKISP1_MI_CTRL_SP_WRITE_SPLA BIT(24) +#define RKISP1_MI_CTRL_SP_WRITE_INT (2 << 24) +#define RKISP1_MI_CTRL_SP_INPUT_YUV400 (0 << 26) +#define RKISP1_MI_CTRL_SP_INPUT_YUV420 BIT(26) +#define RKISP1_MI_CTRL_SP_INPUT_YUV422 (2 << 26) +#define RKISP1_MI_CTRL_SP_INPUT_YUV444 (3 << 26) +#define RKISP1_MI_CTRL_SP_OUTPUT_YUV400 (0 << 28) +#define RKISP1_MI_CTRL_SP_OUTPUT_YUV420 BIT(28) +#define RKISP1_MI_CTRL_SP_OUTPUT_YUV422 (2 << 28) +#define RKISP1_MI_CTRL_SP_OUTPUT_YUV444 (3 << 28) +#define RKISP1_MI_CTRL_SP_OUTPUT_RGB565 (4 << 28) +#define RKISP1_MI_CTRL_SP_OUTPUT_RGB666 (5 << 28) +#define RKISP1_MI_CTRL_SP_OUTPUT_RGB888 (6 << 28) + +#define RKISP1_MI_CTRL_MP_FMT_MASK GENMASK(23, 22) +#define RKISP1_MI_CTRL_SP_FMT_MASK GENMASK(30, 24) + +/* MI_INIT */ +#define RKISP1_CIF_MI_INIT_SKIP BIT(2) +#define RKISP1_CIF_MI_INIT_SOFT_UPD BIT(4) + +/* MI_CTRL_SHD */ +#define RKISP1_CIF_MI_CTRL_SHD_MP_IN_ENABLED BIT(0) +#define RKISP1_CIF_MI_CTRL_SHD_SP_IN_ENABLED BIT(1) +#define RKISP1_CIF_MI_CTRL_SHD_JPEG_IN_ENABLED BIT(2) +#define RKISP1_CIF_MI_CTRL_SHD_RAW_IN_ENABLED BIT(3) +#define RKISP1_CIF_MI_CTRL_SHD_MP_OUT_ENABLED BIT(16) +#define RKISP1_CIF_MI_CTRL_SHD_SP_OUT_ENABLED BIT(17) +#define RKISP1_CIF_MI_CTRL_SHD_JPEG_OUT_ENABLED BIT(18) +#define RKISP1_CIF_MI_CTRL_SHD_RAW_OUT_ENABLED BIT(19) + +/* RSZ_CTRL */ +#define RKISP1_CIF_RSZ_CTRL_SCALE_HY_ENABLE BIT(0) +#define RKISP1_CIF_RSZ_CTRL_SCALE_HC_ENABLE BIT(1) +#define RKISP1_CIF_RSZ_CTRL_SCALE_VY_ENABLE BIT(2) +#define RKISP1_CIF_RSZ_CTRL_SCALE_VC_ENABLE BIT(3) +#define RKISP1_CIF_RSZ_CTRL_SCALE_HY_UP BIT(4) +#define RKISP1_CIF_RSZ_CTRL_SCALE_HC_UP BIT(5) +#define RKISP1_CIF_RSZ_CTRL_SCALE_VY_UP BIT(6) +#define RKISP1_CIF_RSZ_CTRL_SCALE_VC_UP BIT(7) +#define RKISP1_CIF_RSZ_CTRL_CFG_UPD BIT(8) +#define RKISP1_CIF_RSZ_CTRL_CFG_UPD_AUTO BIT(9) +#define RKISP1_CIF_RSZ_SCALER_FACTOR BIT(16) + +/* MI_IMSC - MI_MIS - MI_RIS - MI_ICR - MI_ISR */ +#define RKISP1_CIF_MI_FRAME(stream) BIT((stream)->id) +#define RKISP1_CIF_MI_MBLK_LINE BIT(2) +#define RKISP1_CIF_MI_FILL_MP_Y BIT(3) +#define RKISP1_CIF_MI_WRAP_MP_Y BIT(4) +#define RKISP1_CIF_MI_WRAP_MP_CB BIT(5) +#define RKISP1_CIF_MI_WRAP_MP_CR BIT(6) +#define RKISP1_CIF_MI_WRAP_SP_Y BIT(7) +#define RKISP1_CIF_MI_WRAP_SP_CB BIT(8) +#define RKISP1_CIF_MI_WRAP_SP_CR BIT(9) +#define RKISP1_CIF_MI_DMA_READY BIT(11) + +/* MI_STATUS */ +#define RKISP1_CIF_MI_STATUS_MP_Y_FIFO_FULL BIT(0) +#define RKISP1_CIF_MI_STATUS_SP_Y_FIFO_FULL BIT(4) + +/* MI_DMA_CTRL */ +#define RKISP1_CIF_MI_DMA_CTRL_BURST_LEN_LUM_16 (0 << 0) +#define RKISP1_CIF_MI_DMA_CTRL_BURST_LEN_LUM_32 BIT(0) +#define RKISP1_CIF_MI_DMA_CTRL_BURST_LEN_LUM_64 (2 << 0) +#define RKISP1_CIF_MI_DMA_CTRL_BURST_LEN_CHROM_16 (0 << 2) +#define RKISP1_CIF_MI_DMA_CTRL_BURST_LEN_CHROM_32 BIT(2) +#define RKISP1_CIF_MI_DMA_CTRL_BURST_LEN_CHROM_64 (2 << 2) +#define RKISP1_CIF_MI_DMA_CTRL_READ_FMT_PLANAR (0 << 4) +#define RKISP1_CIF_MI_DMA_CTRL_READ_FMT_SPLANAR BIT(4) +#define RKISP1_CIF_MI_DMA_CTRL_FMT_YUV400 (0 << 6) +#define RKISP1_CIF_MI_DMA_CTRL_FMT_YUV420 BIT(6) +#define RKISP1_CIF_MI_DMA_CTRL_READ_FMT_PACKED (2 << 4) +#define RKISP1_CIF_MI_DMA_CTRL_FMT_YUV422 (2 << 6) +#define RKISP1_CIF_MI_DMA_CTRL_FMT_YUV444 (3 << 6) +#define RKISP1_CIF_MI_DMA_CTRL_BYTE_SWAP BIT(8) +#define RKISP1_CIF_MI_DMA_CTRL_CONTINUOUS_ENA BIT(9) +#define RKISP1_CIF_MI_DMA_CTRL_RGB_BAYER_NO (0 << 12) +#define RKISP1_CIF_MI_DMA_CTRL_RGB_BAYER_8BIT BIT(12) +#define RKISP1_CIF_MI_DMA_CTRL_RGB_BAYER_16BIT (2 << 12) +/* MI_DMA_START */ +#define RKISP1_CIF_MI_DMA_START_ENABLE BIT(0) +/* MI_XTD_FORMAT_CTRL */ +#define RKISP1_CIF_MI_XTD_FMT_CTRL_MP_CB_CR_SWAP BIT(0) +#define RKISP1_CIF_MI_XTD_FMT_CTRL_SP_CB_CR_SWAP BIT(1) +#define RKISP1_CIF_MI_XTD_FMT_CTRL_DMA_CB_CR_SWAP BIT(2) + +/* CCL */ +#define RKISP1_CIF_CCL_CIF_CLK_DIS BIT(2) +/* ICCL */ +#define RKISP1_CIF_ICCL_ISP_CLK BIT(0) +#define RKISP1_CIF_ICCL_CP_CLK BIT(1) +#define RKISP1_CIF_ICCL_RES_2 BIT(2) +#define RKISP1_CIF_ICCL_MRSZ_CLK BIT(3) +#define RKISP1_CIF_ICCL_SRSZ_CLK BIT(4) +#define RKISP1_CIF_ICCL_JPEG_CLK BIT(5) +#define RKISP1_CIF_ICCL_MI_CLK BIT(6) +#define RKISP1_CIF_ICCL_RES_7 BIT(7) +#define RKISP1_CIF_ICCL_IE_CLK BIT(8) +#define RKISP1_CIF_ICCL_SIMP_CLK BIT(9) +#define RKISP1_CIF_ICCL_SMIA_CLK BIT(10) +#define RKISP1_CIF_ICCL_MIPI_CLK BIT(11) +#define RKISP1_CIF_ICCL_DCROP_CLK BIT(12) +/* IRCL */ +#define RKISP1_CIF_IRCL_ISP_SW_RST BIT(0) +#define RKISP1_CIF_IRCL_CP_SW_RST BIT(1) +#define RKISP1_CIF_IRCL_YCS_SW_RST BIT(2) +#define RKISP1_CIF_IRCL_MRSZ_SW_RST BIT(3) +#define RKISP1_CIF_IRCL_SRSZ_SW_RST BIT(4) +#define RKISP1_CIF_IRCL_JPEG_SW_RST BIT(5) +#define RKISP1_CIF_IRCL_MI_SW_RST BIT(6) +#define RKISP1_CIF_IRCL_CIF_SW_RST BIT(7) +#define RKISP1_CIF_IRCL_IE_SW_RST BIT(8) +#define RKISP1_CIF_IRCL_SI_SW_RST BIT(9) +#define RKISP1_CIF_IRCL_MIPI_SW_RST BIT(11) + +/* C_PROC_CTR */ +#define RKISP1_CIF_C_PROC_CTR_ENABLE BIT(0) +#define RKISP1_CIF_C_PROC_YOUT_FULL BIT(1) +#define RKISP1_CIF_C_PROC_YIN_FULL BIT(2) +#define RKISP1_CIF_C_PROC_COUT_FULL BIT(3) +#define RKISP1_CIF_C_PROC_CTRL_RESERVED 0xFFFFFFFE +#define RKISP1_CIF_C_PROC_CONTRAST_RESERVED 0xFFFFFF00 +#define RKISP1_CIF_C_PROC_BRIGHTNESS_RESERVED 0xFFFFFF00 +#define RKISP1_CIF_C_PROC_HUE_RESERVED 0xFFFFFF00 +#define RKISP1_CIF_C_PROC_SATURATION_RESERVED 0xFFFFFF00 +#define RKISP1_CIF_C_PROC_MACC_RESERVED 0xE000E000 +#define RKISP1_CIF_C_PROC_TONE_RESERVED 0xF000 +/* DUAL_CROP_CTRL */ +#define RKISP1_CIF_DUAL_CROP_MP_MODE_BYPASS (0 << 0) +#define RKISP1_CIF_DUAL_CROP_MP_MODE_YUV BIT(0) +#define RKISP1_CIF_DUAL_CROP_MP_MODE_RAW (2 << 0) +#define RKISP1_CIF_DUAL_CROP_SP_MODE_BYPASS (0 << 2) +#define RKISP1_CIF_DUAL_CROP_SP_MODE_YUV BIT(2) +#define RKISP1_CIF_DUAL_CROP_SP_MODE_RAW (2 << 2) +#define RKISP1_CIF_DUAL_CROP_CFG_UPD_PERMANENT BIT(4) +#define RKISP1_CIF_DUAL_CROP_CFG_UPD BIT(5) +#define RKISP1_CIF_DUAL_CROP_GEN_CFG_UPD BIT(6) + +/* IMG_EFF_CTRL */ +#define RKISP1_CIF_IMG_EFF_CTRL_ENABLE BIT(0) +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_BLACKWHITE (0 << 1) +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_NEGATIVE BIT(1) +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_SEPIA (2 << 1) +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_COLOR_SEL (3 << 1) +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_EMBOSS (4 << 1) +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_SKETCH (5 << 1) +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_SHARPEN (6 << 1) +#define RKISP1_CIF_IMG_EFF_CTRL_CFG_UPD BIT(4) +#define RKISP1_CIF_IMG_EFF_CTRL_YCBCR_FULL BIT(5) + +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_BLACKWHITE_SHIFT 0 +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_NEGATIVE_SHIFT 1 +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_SEPIA_SHIFT 2 +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_COLOR_SEL_SHIFT 3 +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_EMBOSS_SHIFT 4 +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_SKETCH_SHIFT 5 +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_SHARPEN_SHIFT 6 +#define RKISP1_CIF_IMG_EFF_CTRL_MODE_MASK 0xE + +/* IMG_EFF_COLOR_SEL */ +#define RKISP1_CIF_IMG_EFF_COLOR_RGB 0 +#define RKISP1_CIF_IMG_EFF_COLOR_B BIT(0) +#define RKISP1_CIF_IMG_EFF_COLOR_G (2 << 0) +#define RKISP1_CIF_IMG_EFF_COLOR_GB (3 << 0) +#define RKISP1_CIF_IMG_EFF_COLOR_R (4 << 0) +#define RKISP1_CIF_IMG_EFF_COLOR_RB (5 << 0) +#define RKISP1_CIF_IMG_EFF_COLOR_RG (6 << 0) +#define RKISP1_CIF_IMG_EFF_COLOR_RGB2 (7 << 0) + +/* MIPI_CTRL */ +#define RKISP1_CIF_MIPI_CTRL_OUTPUT_ENA BIT(0) +#define RKISP1_CIF_MIPI_CTRL_SHUTDOWNLANES(a) (((a) & 0xF) << 8) +#define RKISP1_CIF_MIPI_CTRL_NUM_LANES(a) (((a) & 0x3) << 12) +#define RKISP1_CIF_MIPI_CTRL_ERR_SOT_HS_SKIP BIT(16) +#define RKISP1_CIF_MIPI_CTRL_ERR_SOT_SYNC_HS_SKIP BIT(17) +#define RKISP1_CIF_MIPI_CTRL_CLOCKLANE_ENA BIT(18) + +/* MIPI_DATA_SEL */ +#define RKISP1_CIF_MIPI_DATA_SEL_VC(a) (((a) & 0x3) << 6) +#define RKISP1_CIF_MIPI_DATA_SEL_DT(a) (((a) & 0x3F) << 0) +/* MIPI DATA_TYPE */ +#define RKISP1_CIF_CSI2_DT_YUV420_8b 0x18 +#define RKISP1_CIF_CSI2_DT_YUV420_10b 0x19 +#define RKISP1_CIF_CSI2_DT_YUV422_8b 0x1E +#define RKISP1_CIF_CSI2_DT_YUV422_10b 0x1F +#define RKISP1_CIF_CSI2_DT_RGB565 0x22 +#define RKISP1_CIF_CSI2_DT_RGB666 0x23 +#define RKISP1_CIF_CSI2_DT_RGB888 0x24 +#define RKISP1_CIF_CSI2_DT_RAW8 0x2A +#define RKISP1_CIF_CSI2_DT_RAW10 0x2B +#define RKISP1_CIF_CSI2_DT_RAW12 0x2C + +/* MIPI_IMSC, MIPI_RIS, MIPI_MIS, MIPI_ICR, MIPI_ISR */ +#define RKISP1_CIF_MIPI_SYNC_FIFO_OVFLW(a) (((a) & 0xF) << 0) +#define RKISP1_CIF_MIPI_ERR_SOT(a) (((a) & 0xF) << 4) +#define RKISP1_CIF_MIPI_ERR_SOT_SYNC(a) (((a) & 0xF) << 8) +#define RKISP1_CIF_MIPI_ERR_EOT_SYNC(a) (((a) & 0xF) << 12) +#define RKISP1_CIF_MIPI_ERR_CTRL(a) (((a) & 0xF) << 16) +#define RKISP1_CIF_MIPI_ERR_PROTOCOL BIT(20) +#define RKISP1_CIF_MIPI_ERR_ECC1 BIT(21) +#define RKISP1_CIF_MIPI_ERR_ECC2 BIT(22) +#define RKISP1_CIF_MIPI_ERR_CS BIT(23) +#define RKISP1_CIF_MIPI_FRAME_END BIT(24) +#define RKISP1_CIF_MIPI_ADD_DATA_OVFLW BIT(25) +#define RKISP1_CIF_MIPI_ADD_DATA_WATER_MARK BIT(26) + +#define RKISP1_CIF_MIPI_ERR_CSI (RKISP1_CIF_MIPI_ERR_PROTOCOL | \ + RKISP1_CIF_MIPI_ERR_ECC1 | \ + RKISP1_CIF_MIPI_ERR_ECC2 | \ + RKISP1_CIF_MIPI_ERR_CS) + +#define RKISP1_CIF_MIPI_ERR_DPHY (RKISP1_CIF_MIPI_ERR_SOT(3) | \ + RKISP1_CIF_MIPI_ERR_SOT_SYNC(3) | \ + RKISP1_CIF_MIPI_ERR_EOT_SYNC(3) | \ + RKISP1_CIF_MIPI_ERR_CTRL(3)) + +/* SUPER_IMPOSE */ +#define RKISP1_CIF_SUPER_IMP_CTRL_NORMAL_MODE BIT(0) +#define RKISP1_CIF_SUPER_IMP_CTRL_REF_IMG_MEM BIT(1) +#define RKISP1_CIF_SUPER_IMP_CTRL_TRANSP_DIS BIT(2) + +/* ISP HISTOGRAM CALCULATION : ISP_HIST_PROP */ +#define RKISP1_CIF_ISP_HIST_PROP_MODE_DIS (0 << 0) +#define RKISP1_CIF_ISP_HIST_PROP_MODE_RGB BIT(0) +#define RKISP1_CIF_ISP_HIST_PROP_MODE_RED (2 << 0) +#define RKISP1_CIF_ISP_HIST_PROP_MODE_GREEN (3 << 0) +#define RKISP1_CIF_ISP_HIST_PROP_MODE_BLUE (4 << 0) +#define RKISP1_CIF_ISP_HIST_PROP_MODE_LUM (5 << 0) +#define RKISP1_CIF_ISP_HIST_PROP_MODE_MASK 0x7 +#define RKISP1_CIF_ISP_HIST_PREDIV_SET(x) (((x) & 0x7F) << 3) +#define RKISP1_CIF_ISP_HIST_WEIGHT_SET(v0, v1, v2, v3) \ + (((v0) & 0x1F) | (((v1) & 0x1F) << 8) |\ + (((v2) & 0x1F) << 16) | \ + (((v3) & 0x1F) << 24)) + +#define RKISP1_CIF_ISP_HIST_WINDOW_OFFSET_RESERVED 0xFFFFF000 +#define RKISP1_CIF_ISP_HIST_WINDOW_SIZE_RESERVED 0xFFFFF800 +#define RKISP1_CIF_ISP_HIST_WEIGHT_RESERVED 0xE0E0E0E0 +#define RKISP1_CIF_ISP_MAX_HIST_PREDIVIDER 0x0000007F +#define RKISP1_CIF_ISP_HIST_ROW_NUM 5 +#define RKISP1_CIF_ISP_HIST_COLUMN_NUM 5 + +/* AUTO FOCUS MEASUREMENT: ISP_AFM_CTRL */ +#define RKISP1_ISP_AFM_CTRL_ENABLE BIT(0) + +/* SHUTTER CONTROL */ +#define RKISP1_CIF_ISP_SH_CTRL_SH_ENA BIT(0) +#define RKISP1_CIF_ISP_SH_CTRL_REP_EN BIT(1) +#define RKISP1_CIF_ISP_SH_CTRL_SRC_SH_TRIG BIT(2) +#define RKISP1_CIF_ISP_SH_CTRL_EDGE_POS BIT(3) +#define RKISP1_CIF_ISP_SH_CTRL_POL_LOW BIT(4) + +/* FLASH MODULE */ +/* ISP_FLASH_CMD */ +#define RKISP1_CIFFLASH_CMD_PRELIGHT_ON BIT(0) +#define RKISP1_CIFFLASH_CMD_FLASH_ON BIT(1) +#define RKISP1_CIFFLASH_CMD_PRE_FLASH_ON BIT(2) +/* ISP_FLASH_CONFIG */ +#define RKISP1_CIFFLASH_CONFIG_PRELIGHT_END BIT(0) +#define RKISP1_CIFFLASH_CONFIG_VSYNC_POS BIT(1) +#define RKISP1_CIFFLASH_CONFIG_PRELIGHT_LOW BIT(2) +#define RKISP1_CIFFLASH_CONFIG_SRC_FL_TRIG BIT(3) +#define RKISP1_CIFFLASH_CONFIG_DELAY(a) (((a) & 0xF) << 4) + +/* Demosaic: ISP_DEMOSAIC */ +#define RKISP1_CIF_ISP_DEMOSAIC_BYPASS BIT(10) +#define RKISP1_CIF_ISP_DEMOSAIC_TH(x) ((x) & 0xFF) + +/* AWB */ +/* ISP_AWB_PROP */ +#define RKISP1_CIF_ISP_AWB_YMAX_CMP_EN BIT(2) +#define RKISP1_CIF_ISP_AWB_YMAX_READ(x) (((x) >> 2) & 1) +#define RKISP1_CIF_ISP_AWB_MODE_RGB_EN ((1 << 31) | (0x2 << 0)) +#define RKISP1_CIF_ISP_AWB_MODE_YCBCR_EN ((0 << 31) | (0x2 << 0)) +#define RKISP1_CIF_ISP_AWB_MODE_YCBCR_EN ((0 << 31) | (0x2 << 0)) +#define RKISP1_CIF_ISP_AWB_MODE_MASK_NONE 0xFFFFFFFC +#define RKISP1_CIF_ISP_AWB_MODE_READ(x) ((x) & 3) +/* ISP_AWB_GAIN_RB, ISP_AWB_GAIN_G */ +#define RKISP1_CIF_ISP_AWB_GAIN_R_SET(x) (((x) & 0x3FF) << 16) +#define RKISP1_CIF_ISP_AWB_GAIN_R_READ(x) (((x) >> 16) & 0x3FF) +#define RKISP1_CIF_ISP_AWB_GAIN_B_SET(x) ((x) & 0x3FFF) +#define RKISP1_CIF_ISP_AWB_GAIN_B_READ(x) ((x) & 0x3FFF) +/* ISP_AWB_REF */ +#define RKISP1_CIF_ISP_AWB_REF_CR_SET(x) (((x) & 0xFF) << 8) +#define RKISP1_CIF_ISP_AWB_REF_CR_READ(x) (((x) >> 8) & 0xFF) +#define RKISP1_CIF_ISP_AWB_REF_CB_READ(x) ((x) & 0xFF) +/* ISP_AWB_THRESH */ +#define RKISP1_CIF_ISP_AWB_MAX_CS_SET(x) (((x) & 0xFF) << 8) +#define RKISP1_CIF_ISP_AWB_MAX_CS_READ(x) (((x) >> 8) & 0xFF) +#define RKISP1_CIF_ISP_AWB_MIN_C_READ(x) ((x) & 0xFF) +#define RKISP1_CIF_ISP_AWB_MIN_Y_SET(x) (((x) & 0xFF) << 16) +#define RKISP1_CIF_ISP_AWB_MIN_Y_READ(x) (((x) >> 16) & 0xFF) +#define RKISP1_CIF_ISP_AWB_MAX_Y_SET(x) (((x) & 0xFF) << 24) +#define RKISP1_CIF_ISP_AWB_MAX_Y_READ(x) (((x) >> 24) & 0xFF) +/* ISP_AWB_MEAN */ +#define RKISP1_CIF_ISP_AWB_GET_MEAN_CR_R(x) ((x) & 0xFF) +#define RKISP1_CIF_ISP_AWB_GET_MEAN_CB_B(x) (((x) >> 8) & 0xFF) +#define RKISP1_CIF_ISP_AWB_GET_MEAN_Y_G(x) (((x) >> 16) & 0xFF) +/* ISP_AWB_WHITE_CNT */ +#define RKISP1_CIF_ISP_AWB_GET_PIXEL_CNT(x) ((x) & 0x3FFFFFF) + +#define RKISP1_CIF_ISP_AWB_GAINS_MAX_VAL 0x000003FF +#define RKISP1_CIF_ISP_AWB_WINDOW_OFFSET_MAX 0x00000FFF +#define RKISP1_CIF_ISP_AWB_WINDOW_MAX_SIZE 0x00001FFF +#define RKISP1_CIF_ISP_AWB_CBCR_MAX_REF 0x000000FF +#define RKISP1_CIF_ISP_AWB_THRES_MAX_YC 0x000000FF + +/* AE */ +/* ISP_EXP_CTRL */ +#define RKISP1_CIF_ISP_EXP_ENA BIT(0) +#define RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP BIT(1) +/* + *'1' luminance calculation according to Y=(R+G+B) x 0.332 (85/256) + *'0' luminance calculation according to Y=16+0.25R+0.5G+0.1094B + */ +#define RKISP1_CIF_ISP_EXP_CTRL_MEASMODE_1 BIT(31) + +/* ISP_EXP_H_SIZE */ +#define RKISP1_CIF_ISP_EXP_H_SIZE_SET(x) ((x) & 0x7FF) +#define RKISP1_CIF_ISP_EXP_HEIGHT_MASK 0x000007FF +/* ISP_EXP_V_SIZE : vertical size must be a multiple of 2). */ +#define RKISP1_CIF_ISP_EXP_V_SIZE_SET(x) ((x) & 0x7FE) + +/* ISP_EXP_H_OFFSET */ +#define RKISP1_CIF_ISP_EXP_H_OFFSET_SET(x) ((x) & 0x1FFF) +#define RKISP1_CIF_ISP_EXP_MAX_HOFFS 2424 +/* ISP_EXP_V_OFFSET */ +#define RKISP1_CIF_ISP_EXP_V_OFFSET_SET(x) ((x) & 0x1FFF) +#define RKISP1_CIF_ISP_EXP_MAX_VOFFS 1806 + +#define RKISP1_CIF_ISP_EXP_ROW_NUM 5 +#define RKISP1_CIF_ISP_EXP_COLUMN_NUM 5 +#define RKISP1_CIF_ISP_EXP_NUM_LUMA_REGS \ + (RKISP1_CIF_ISP_EXP_ROW_NUM * RKISP1_CIF_ISP_EXP_COLUMN_NUM) +#define RKISP1_CIF_ISP_EXP_BLOCK_MAX_HSIZE 516 +#define RKISP1_CIF_ISP_EXP_BLOCK_MIN_HSIZE 35 +#define RKISP1_CIF_ISP_EXP_BLOCK_MAX_VSIZE 390 +#define RKISP1_CIF_ISP_EXP_BLOCK_MIN_VSIZE 28 +#define RKISP1_CIF_ISP_EXP_MAX_HSIZE \ + (RKISP1_CIF_ISP_EXP_BLOCK_MAX_HSIZE * RKISP1_CIF_ISP_EXP_COLUMN_NUM + 1) +#define RKISP1_CIF_ISP_EXP_MIN_HSIZE \ + (RKISP1_CIF_ISP_EXP_BLOCK_MIN_HSIZE * RKISP1_CIF_ISP_EXP_COLUMN_NUM + 1) +#define RKISP1_CIF_ISP_EXP_MAX_VSIZE \ + (RKISP1_CIF_ISP_EXP_BLOCK_MAX_VSIZE * RKISP1_CIF_ISP_EXP_ROW_NUM + 1) +#define RKISP1_CIF_ISP_EXP_MIN_VSIZE \ + (RKISP1_CIF_ISP_EXP_BLOCK_MIN_VSIZE * RKISP1_CIF_ISP_EXP_ROW_NUM + 1) + +/* LSC: ISP_LSC_CTRL */ +#define RKISP1_CIF_ISP_LSC_CTRL_ENA BIT(0) +#define RKISP1_CIF_ISP_LSC_SECT_SIZE_RESERVED 0xFC00FC00 +#define RKISP1_CIF_ISP_LSC_GRAD_RESERVED 0xF000F000 +#define RKISP1_CIF_ISP_LSC_SAMPLE_RESERVED 0xF000F000 +#define RKISP1_CIF_ISP_LSC_SECTORS_MAX 17 +#define RKISP1_CIF_ISP_LSC_TABLE_DATA(v0, v1) \ + (((v0) & 0xFFF) | (((v1) & 0xFFF) << 12)) +#define RKISP1_CIF_ISP_LSC_SECT_SIZE(v0, v1) \ + (((v0) & 0xFFF) | (((v1) & 0xFFF) << 16)) +#define RKISP1_CIF_ISP_LSC_GRAD_SIZE(v0, v1) \ + (((v0) & 0xFFF) | (((v1) & 0xFFF) << 16)) + +/* LSC: ISP_LSC_TABLE_SEL */ +#define RKISP1_CIF_ISP_LSC_TABLE_0 0 +#define RKISP1_CIF_ISP_LSC_TABLE_1 1 + +/* LSC: ISP_LSC_STATUS */ +#define RKISP1_CIF_ISP_LSC_ACTIVE_TABLE BIT(1) +#define RKISP1_CIF_ISP_LSC_TABLE_ADDRESS_0 0 +#define RKISP1_CIF_ISP_LSC_TABLE_ADDRESS_153 153 + +/* FLT */ +/* ISP_FILT_MODE */ +#define RKISP1_CIF_ISP_FLT_ENA BIT(0) + +/* + * 0: green filter static mode (active filter factor = FILT_FAC_MID) + * 1: dynamic noise reduction/sharpen Default + */ +#define RKISP1_CIF_ISP_FLT_MODE_DNR BIT(1) +#define RKISP1_CIF_ISP_FLT_MODE_MAX 1 +#define RKISP1_CIF_ISP_FLT_CHROMA_V_MODE(x) (((x) & 0x3) << 4) +#define RKISP1_CIF_ISP_FLT_CHROMA_H_MODE(x) (((x) & 0x3) << 6) +#define RKISP1_CIF_ISP_FLT_CHROMA_MODE_MAX 3 +#define RKISP1_CIF_ISP_FLT_GREEN_STAGE1(x) (((x) & 0xF) << 8) +#define RKISP1_CIF_ISP_FLT_GREEN_STAGE1_MAX 8 +#define RKISP1_CIF_ISP_FLT_THREAD_RESERVED 0xFFFFFC00 +#define RKISP1_CIF_ISP_FLT_FAC_RESERVED 0xFFFFFFC0 +#define RKISP1_CIF_ISP_FLT_LUM_WEIGHT_RESERVED 0xFFF80000 + +#define RKISP1_CIF_ISP_CTK_COEFF_RESERVED 0xFFFFF800 +#define RKISP1_CIF_ISP_XTALK_OFFSET_RESERVED 0xFFFFF000 + +/* GOC */ +#define RKISP1_CIF_ISP_GAMMA_OUT_MODE_EQU BIT(0) +#define RKISP1_CIF_ISP_GOC_MODE_MAX 1 +#define RKISP1_CIF_ISP_GOC_RESERVED 0xFFFFF800 +/* ISP_CTRL BIT 11*/ +#define RKISP1_CIF_ISP_CTRL_ISP_GAMMA_OUT_ENA_READ(x) (((x) >> 11) & 1) + +/* DPCC */ +/* ISP_DPCC_MODE */ +#define RKISP1_CIF_ISP_DPCC_ENA BIT(0) +#define RKISP1_CIF_ISP_DPCC_MODE_MAX 0x07 +#define RKISP1_CIF_ISP_DPCC_OUTPUTMODE_MAX 0x0F +#define RKISP1_CIF_ISP_DPCC_SETUSE_MAX 0x0F +#define RKISP1_CIF_ISP_DPCC_METHODS_SET_RESERVED 0xFFFFE000 +#define RKISP1_CIF_ISP_DPCC_LINE_THRESH_RESERVED 0xFFFF0000 +#define RKISP1_CIF_ISP_DPCC_LINE_MAD_FAC_RESERVED 0xFFFFC0C0 +#define RKISP1_CIF_ISP_DPCC_PG_FAC_RESERVED 0xFFFFC0C0 +#define RKISP1_CIF_ISP_DPCC_RND_THRESH_RESERVED 0xFFFF0000 +#define RKISP1_CIF_ISP_DPCC_RG_FAC_RESERVED 0xFFFFC0C0 +#define RKISP1_CIF_ISP_DPCC_RO_LIMIT_RESERVED 0xFFFFF000 +#define RKISP1_CIF_ISP_DPCC_RND_OFFS_RESERVED 0xFFFFF000 + +/* BLS */ +/* ISP_BLS_CTRL */ +#define RKISP1_CIF_ISP_BLS_ENA BIT(0) +#define RKISP1_CIF_ISP_BLS_MODE_MEASURED BIT(1) +#define RKISP1_CIF_ISP_BLS_MODE_FIXED 0 +#define RKISP1_CIF_ISP_BLS_WINDOW_1 BIT(2) +#define RKISP1_CIF_ISP_BLS_WINDOW_2 (2 << 2) + +/* GAMMA-IN */ +#define RKISP1_CIFISP_DEGAMMA_X_RESERVED \ + ((1 << 31) | (1 << 27) | (1 << 23) | (1 << 19) |\ + (1 << 15) | (1 << 11) | (1 << 7) | (1 << 3)) +#define RKISP1_CIFISP_DEGAMMA_Y_RESERVED 0xFFFFF000 + +/* AFM */ +#define RKISP1_CIF_ISP_AFM_ENA BIT(0) +#define RKISP1_CIF_ISP_AFM_THRES_RESERVED 0xFFFF0000 +#define RKISP1_CIF_ISP_AFM_VAR_SHIFT_RESERVED 0xFFF8FFF8 +#define RKISP1_CIF_ISP_AFM_WINDOW_X_RESERVED 0xE000 +#define RKISP1_CIF_ISP_AFM_WINDOW_Y_RESERVED 0xF000 +#define RKISP1_CIF_ISP_AFM_WINDOW_X_MIN 0x5 +#define RKISP1_CIF_ISP_AFM_WINDOW_Y_MIN 0x2 +#define RKISP1_CIF_ISP_AFM_WINDOW_X(x) (((x) & 0x1FFF) << 16) +#define RKISP1_CIF_ISP_AFM_WINDOW_Y(x) ((x) & 0x1FFF) + +/* DPF */ +#define RKISP1_CIF_ISP_DPF_MODE_EN BIT(0) +#define RKISP1_CIF_ISP_DPF_MODE_B_FLT_DIS BIT(1) +#define RKISP1_CIF_ISP_DPF_MODE_GB_FLT_DIS BIT(2) +#define RKISP1_CIF_ISP_DPF_MODE_GR_FLT_DIS BIT(3) +#define RKISP1_CIF_ISP_DPF_MODE_R_FLT_DIS BIT(4) +#define RKISP1_CIF_ISP_DPF_MODE_RB_FLTSIZE_9x9 BIT(5) +#define RKISP1_CIF_ISP_DPF_MODE_NLL_SEGMENTATION BIT(6) +#define RKISP1_CIF_ISP_DPF_MODE_AWB_GAIN_COMP BIT(7) +#define RKISP1_CIF_ISP_DPF_MODE_LSC_GAIN_COMP BIT(8) +#define RKISP1_CIF_ISP_DPF_MODE_USE_NF_GAIN BIT(9) +#define RKISP1_CIF_ISP_DPF_NF_GAIN_RESERVED 0xFFFFF000 +#define RKISP1_CIF_ISP_DPF_SPATIAL_COEFF_MAX 0x1F +#define RKISP1_CIF_ISP_DPF_NLL_COEFF_N_MAX 0x3FF + +/* =================================================================== */ +/* CIF Registers */ +/* =================================================================== */ +#define RKISP1_CIF_CTRL_BASE 0x00000000 +#define RKISP1_CIF_CCL (RKISP1_CIF_CTRL_BASE + 0x00000000) +#define RKISP1_CIF_VI_ID (RKISP1_CIF_CTRL_BASE + 0x00000008) +#define RKISP1_CIF_ICCL (RKISP1_CIF_CTRL_BASE + 0x00000010) +#define RKISP1_CIF_IRCL (RKISP1_CIF_CTRL_BASE + 0x00000014) +#define RKISP1_CIF_VI_DPCL (RKISP1_CIF_CTRL_BASE + 0x00000018) + +#define RKISP1_CIF_IMG_EFF_BASE 0x00000200 +#define RKISP1_CIF_IMG_EFF_CTRL (RKISP1_CIF_IMG_EFF_BASE + 0x00000000) +#define RKISP1_CIF_IMG_EFF_COLOR_SEL (RKISP1_CIF_IMG_EFF_BASE + 0x00000004) +#define RKISP1_CIF_IMG_EFF_MAT_1 (RKISP1_CIF_IMG_EFF_BASE + 0x00000008) +#define RKISP1_CIF_IMG_EFF_MAT_2 (RKISP1_CIF_IMG_EFF_BASE + 0x0000000C) +#define RKISP1_CIF_IMG_EFF_MAT_3 (RKISP1_CIF_IMG_EFF_BASE + 0x00000010) +#define RKISP1_CIF_IMG_EFF_MAT_4 (RKISP1_CIF_IMG_EFF_BASE + 0x00000014) +#define RKISP1_CIF_IMG_EFF_MAT_5 (RKISP1_CIF_IMG_EFF_BASE + 0x00000018) +#define RKISP1_CIF_IMG_EFF_TINT (RKISP1_CIF_IMG_EFF_BASE + 0x0000001C) +#define RKISP1_CIF_IMG_EFF_CTRL_SHD (RKISP1_CIF_IMG_EFF_BASE + 0x00000020) +#define RKISP1_CIF_IMG_EFF_SHARPEN (RKISP1_CIF_IMG_EFF_BASE + 0x00000024) + +#define RKISP1_CIF_SUPER_IMP_BASE 0x00000300 +#define RKISP1_CIF_SUPER_IMP_CTRL (RKISP1_CIF_SUPER_IMP_BASE + 0x00000000) +#define RKISP1_CIF_SUPER_IMP_OFFSET_X (RKISP1_CIF_SUPER_IMP_BASE + 0x00000004) +#define RKISP1_CIF_SUPER_IMP_OFFSET_Y (RKISP1_CIF_SUPER_IMP_BASE + 0x00000008) +#define RKISP1_CIF_SUPER_IMP_COLOR_Y (RKISP1_CIF_SUPER_IMP_BASE + 0x0000000C) +#define RKISP1_CIF_SUPER_IMP_COLOR_CB (RKISP1_CIF_SUPER_IMP_BASE + 0x00000010) +#define RKISP1_CIF_SUPER_IMP_COLOR_CR (RKISP1_CIF_SUPER_IMP_BASE + 0x00000014) + +#define RKISP1_CIF_ISP_BASE 0x00000400 +#define RKISP1_CIF_ISP_CTRL (RKISP1_CIF_ISP_BASE + 0x00000000) +#define RKISP1_CIF_ISP_ACQ_PROP (RKISP1_CIF_ISP_BASE + 0x00000004) +#define RKISP1_CIF_ISP_ACQ_H_OFFS (RKISP1_CIF_ISP_BASE + 0x00000008) +#define RKISP1_CIF_ISP_ACQ_V_OFFS (RKISP1_CIF_ISP_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_ACQ_H_SIZE (RKISP1_CIF_ISP_BASE + 0x00000010) +#define RKISP1_CIF_ISP_ACQ_V_SIZE (RKISP1_CIF_ISP_BASE + 0x00000014) +#define RKISP1_CIF_ISP_ACQ_NR_FRAMES (RKISP1_CIF_ISP_BASE + 0x00000018) +#define RKISP1_CIF_ISP_GAMMA_DX_LO (RKISP1_CIF_ISP_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_GAMMA_DX_HI (RKISP1_CIF_ISP_BASE + 0x00000020) +#define RKISP1_CIF_ISP_GAMMA_R_Y0 (RKISP1_CIF_ISP_BASE + 0x00000024) +#define RKISP1_CIF_ISP_GAMMA_R_Y1 (RKISP1_CIF_ISP_BASE + 0x00000028) +#define RKISP1_CIF_ISP_GAMMA_R_Y2 (RKISP1_CIF_ISP_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_GAMMA_R_Y3 (RKISP1_CIF_ISP_BASE + 0x00000030) +#define RKISP1_CIF_ISP_GAMMA_R_Y4 (RKISP1_CIF_ISP_BASE + 0x00000034) +#define RKISP1_CIF_ISP_GAMMA_R_Y5 (RKISP1_CIF_ISP_BASE + 0x00000038) +#define RKISP1_CIF_ISP_GAMMA_R_Y6 (RKISP1_CIF_ISP_BASE + 0x0000003C) +#define RKISP1_CIF_ISP_GAMMA_R_Y7 (RKISP1_CIF_ISP_BASE + 0x00000040) +#define RKISP1_CIF_ISP_GAMMA_R_Y8 (RKISP1_CIF_ISP_BASE + 0x00000044) +#define RKISP1_CIF_ISP_GAMMA_R_Y9 (RKISP1_CIF_ISP_BASE + 0x00000048) +#define RKISP1_CIF_ISP_GAMMA_R_Y10 (RKISP1_CIF_ISP_BASE + 0x0000004C) +#define RKISP1_CIF_ISP_GAMMA_R_Y11 (RKISP1_CIF_ISP_BASE + 0x00000050) +#define RKISP1_CIF_ISP_GAMMA_R_Y12 (RKISP1_CIF_ISP_BASE + 0x00000054) +#define RKISP1_CIF_ISP_GAMMA_R_Y13 (RKISP1_CIF_ISP_BASE + 0x00000058) +#define RKISP1_CIF_ISP_GAMMA_R_Y14 (RKISP1_CIF_ISP_BASE + 0x0000005C) +#define RKISP1_CIF_ISP_GAMMA_R_Y15 (RKISP1_CIF_ISP_BASE + 0x00000060) +#define RKISP1_CIF_ISP_GAMMA_R_Y16 (RKISP1_CIF_ISP_BASE + 0x00000064) +#define RKISP1_CIF_ISP_GAMMA_G_Y0 (RKISP1_CIF_ISP_BASE + 0x00000068) +#define RKISP1_CIF_ISP_GAMMA_G_Y1 (RKISP1_CIF_ISP_BASE + 0x0000006C) +#define RKISP1_CIF_ISP_GAMMA_G_Y2 (RKISP1_CIF_ISP_BASE + 0x00000070) +#define RKISP1_CIF_ISP_GAMMA_G_Y3 (RKISP1_CIF_ISP_BASE + 0x00000074) +#define RKISP1_CIF_ISP_GAMMA_G_Y4 (RKISP1_CIF_ISP_BASE + 0x00000078) +#define RKISP1_CIF_ISP_GAMMA_G_Y5 (RKISP1_CIF_ISP_BASE + 0x0000007C) +#define RKISP1_CIF_ISP_GAMMA_G_Y6 (RKISP1_CIF_ISP_BASE + 0x00000080) +#define RKISP1_CIF_ISP_GAMMA_G_Y7 (RKISP1_CIF_ISP_BASE + 0x00000084) +#define RKISP1_CIF_ISP_GAMMA_G_Y8 (RKISP1_CIF_ISP_BASE + 0x00000088) +#define RKISP1_CIF_ISP_GAMMA_G_Y9 (RKISP1_CIF_ISP_BASE + 0x0000008C) +#define RKISP1_CIF_ISP_GAMMA_G_Y10 (RKISP1_CIF_ISP_BASE + 0x00000090) +#define RKISP1_CIF_ISP_GAMMA_G_Y11 (RKISP1_CIF_ISP_BASE + 0x00000094) +#define RKISP1_CIF_ISP_GAMMA_G_Y12 (RKISP1_CIF_ISP_BASE + 0x00000098) +#define RKISP1_CIF_ISP_GAMMA_G_Y13 (RKISP1_CIF_ISP_BASE + 0x0000009C) +#define RKISP1_CIF_ISP_GAMMA_G_Y14 (RKISP1_CIF_ISP_BASE + 0x000000A0) +#define RKISP1_CIF_ISP_GAMMA_G_Y15 (RKISP1_CIF_ISP_BASE + 0x000000A4) +#define RKISP1_CIF_ISP_GAMMA_G_Y16 (RKISP1_CIF_ISP_BASE + 0x000000A8) +#define RKISP1_CIF_ISP_GAMMA_B_Y0 (RKISP1_CIF_ISP_BASE + 0x000000AC) +#define RKISP1_CIF_ISP_GAMMA_B_Y1 (RKISP1_CIF_ISP_BASE + 0x000000B0) +#define RKISP1_CIF_ISP_GAMMA_B_Y2 (RKISP1_CIF_ISP_BASE + 0x000000B4) +#define RKISP1_CIF_ISP_GAMMA_B_Y3 (RKISP1_CIF_ISP_BASE + 0x000000B8) +#define RKISP1_CIF_ISP_GAMMA_B_Y4 (RKISP1_CIF_ISP_BASE + 0x000000BC) +#define RKISP1_CIF_ISP_GAMMA_B_Y5 (RKISP1_CIF_ISP_BASE + 0x000000C0) +#define RKISP1_CIF_ISP_GAMMA_B_Y6 (RKISP1_CIF_ISP_BASE + 0x000000C4) +#define RKISP1_CIF_ISP_GAMMA_B_Y7 (RKISP1_CIF_ISP_BASE + 0x000000C8) +#define RKISP1_CIF_ISP_GAMMA_B_Y8 (RKISP1_CIF_ISP_BASE + 0x000000CC) +#define RKISP1_CIF_ISP_GAMMA_B_Y9 (RKISP1_CIF_ISP_BASE + 0x000000D0) +#define RKISP1_CIF_ISP_GAMMA_B_Y10 (RKISP1_CIF_ISP_BASE + 0x000000D4) +#define RKISP1_CIF_ISP_GAMMA_B_Y11 (RKISP1_CIF_ISP_BASE + 0x000000D8) +#define RKISP1_CIF_ISP_GAMMA_B_Y12 (RKISP1_CIF_ISP_BASE + 0x000000DC) +#define RKISP1_CIF_ISP_GAMMA_B_Y13 (RKISP1_CIF_ISP_BASE + 0x000000E0) +#define RKISP1_CIF_ISP_GAMMA_B_Y14 (RKISP1_CIF_ISP_BASE + 0x000000E4) +#define RKISP1_CIF_ISP_GAMMA_B_Y15 (RKISP1_CIF_ISP_BASE + 0x000000E8) +#define RKISP1_CIF_ISP_GAMMA_B_Y16 (RKISP1_CIF_ISP_BASE + 0x000000EC) +#define RKISP1_CIF_ISP_AWB_PROP (RKISP1_CIF_ISP_BASE + 0x00000110) +#define RKISP1_CIF_ISP_AWB_WND_H_OFFS (RKISP1_CIF_ISP_BASE + 0x00000114) +#define RKISP1_CIF_ISP_AWB_WND_V_OFFS (RKISP1_CIF_ISP_BASE + 0x00000118) +#define RKISP1_CIF_ISP_AWB_WND_H_SIZE (RKISP1_CIF_ISP_BASE + 0x0000011C) +#define RKISP1_CIF_ISP_AWB_WND_V_SIZE (RKISP1_CIF_ISP_BASE + 0x00000120) +#define RKISP1_CIF_ISP_AWB_FRAMES (RKISP1_CIF_ISP_BASE + 0x00000124) +#define RKISP1_CIF_ISP_AWB_REF (RKISP1_CIF_ISP_BASE + 0x00000128) +#define RKISP1_CIF_ISP_AWB_THRESH (RKISP1_CIF_ISP_BASE + 0x0000012C) +#define RKISP1_CIF_ISP_AWB_GAIN_G (RKISP1_CIF_ISP_BASE + 0x00000138) +#define RKISP1_CIF_ISP_AWB_GAIN_RB (RKISP1_CIF_ISP_BASE + 0x0000013C) +#define RKISP1_CIF_ISP_AWB_WHITE_CNT (RKISP1_CIF_ISP_BASE + 0x00000140) +#define RKISP1_CIF_ISP_AWB_MEAN (RKISP1_CIF_ISP_BASE + 0x00000144) +#define RKISP1_CIF_ISP_CC_COEFF_0 (RKISP1_CIF_ISP_BASE + 0x00000170) +#define RKISP1_CIF_ISP_CC_COEFF_1 (RKISP1_CIF_ISP_BASE + 0x00000174) +#define RKISP1_CIF_ISP_CC_COEFF_2 (RKISP1_CIF_ISP_BASE + 0x00000178) +#define RKISP1_CIF_ISP_CC_COEFF_3 (RKISP1_CIF_ISP_BASE + 0x0000017C) +#define RKISP1_CIF_ISP_CC_COEFF_4 (RKISP1_CIF_ISP_BASE + 0x00000180) +#define RKISP1_CIF_ISP_CC_COEFF_5 (RKISP1_CIF_ISP_BASE + 0x00000184) +#define RKISP1_CIF_ISP_CC_COEFF_6 (RKISP1_CIF_ISP_BASE + 0x00000188) +#define RKISP1_CIF_ISP_CC_COEFF_7 (RKISP1_CIF_ISP_BASE + 0x0000018C) +#define RKISP1_CIF_ISP_CC_COEFF_8 (RKISP1_CIF_ISP_BASE + 0x00000190) +#define RKISP1_CIF_ISP_OUT_H_OFFS (RKISP1_CIF_ISP_BASE + 0x00000194) +#define RKISP1_CIF_ISP_OUT_V_OFFS (RKISP1_CIF_ISP_BASE + 0x00000198) +#define RKISP1_CIF_ISP_OUT_H_SIZE (RKISP1_CIF_ISP_BASE + 0x0000019C) +#define RKISP1_CIF_ISP_OUT_V_SIZE (RKISP1_CIF_ISP_BASE + 0x000001A0) +#define RKISP1_CIF_ISP_DEMOSAIC (RKISP1_CIF_ISP_BASE + 0x000001A4) +#define RKISP1_CIF_ISP_FLAGS_SHD (RKISP1_CIF_ISP_BASE + 0x000001A8) +#define RKISP1_CIF_ISP_OUT_H_OFFS_SHD (RKISP1_CIF_ISP_BASE + 0x000001AC) +#define RKISP1_CIF_ISP_OUT_V_OFFS_SHD (RKISP1_CIF_ISP_BASE + 0x000001B0) +#define RKISP1_CIF_ISP_OUT_H_SIZE_SHD (RKISP1_CIF_ISP_BASE + 0x000001B4) +#define RKISP1_CIF_ISP_OUT_V_SIZE_SHD (RKISP1_CIF_ISP_BASE + 0x000001B8) +#define RKISP1_CIF_ISP_IMSC (RKISP1_CIF_ISP_BASE + 0x000001BC) +#define RKISP1_CIF_ISP_RIS (RKISP1_CIF_ISP_BASE + 0x000001C0) +#define RKISP1_CIF_ISP_MIS (RKISP1_CIF_ISP_BASE + 0x000001C4) +#define RKISP1_CIF_ISP_ICR (RKISP1_CIF_ISP_BASE + 0x000001C8) +#define RKISP1_CIF_ISP_ISR (RKISP1_CIF_ISP_BASE + 0x000001CC) +#define RKISP1_CIF_ISP_CT_COEFF_0 (RKISP1_CIF_ISP_BASE + 0x000001D0) +#define RKISP1_CIF_ISP_CT_COEFF_1 (RKISP1_CIF_ISP_BASE + 0x000001D4) +#define RKISP1_CIF_ISP_CT_COEFF_2 (RKISP1_CIF_ISP_BASE + 0x000001D8) +#define RKISP1_CIF_ISP_CT_COEFF_3 (RKISP1_CIF_ISP_BASE + 0x000001DC) +#define RKISP1_CIF_ISP_CT_COEFF_4 (RKISP1_CIF_ISP_BASE + 0x000001E0) +#define RKISP1_CIF_ISP_CT_COEFF_5 (RKISP1_CIF_ISP_BASE + 0x000001E4) +#define RKISP1_CIF_ISP_CT_COEFF_6 (RKISP1_CIF_ISP_BASE + 0x000001E8) +#define RKISP1_CIF_ISP_CT_COEFF_7 (RKISP1_CIF_ISP_BASE + 0x000001EC) +#define RKISP1_CIF_ISP_CT_COEFF_8 (RKISP1_CIF_ISP_BASE + 0x000001F0) +#define RKISP1_CIF_ISP_GAMMA_OUT_MODE (RKISP1_CIF_ISP_BASE + 0x000001F4) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_0 (RKISP1_CIF_ISP_BASE + 0x000001F8) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_1 (RKISP1_CIF_ISP_BASE + 0x000001FC) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_2 (RKISP1_CIF_ISP_BASE + 0x00000200) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_3 (RKISP1_CIF_ISP_BASE + 0x00000204) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_4 (RKISP1_CIF_ISP_BASE + 0x00000208) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_5 (RKISP1_CIF_ISP_BASE + 0x0000020C) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_6 (RKISP1_CIF_ISP_BASE + 0x00000210) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_7 (RKISP1_CIF_ISP_BASE + 0x00000214) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_8 (RKISP1_CIF_ISP_BASE + 0x00000218) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_9 (RKISP1_CIF_ISP_BASE + 0x0000021C) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_10 (RKISP1_CIF_ISP_BASE + 0x00000220) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_11 (RKISP1_CIF_ISP_BASE + 0x00000224) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_12 (RKISP1_CIF_ISP_BASE + 0x00000228) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_13 (RKISP1_CIF_ISP_BASE + 0x0000022C) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_14 (RKISP1_CIF_ISP_BASE + 0x00000230) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_15 (RKISP1_CIF_ISP_BASE + 0x00000234) +#define RKISP1_CIF_ISP_GAMMA_OUT_Y_16 (RKISP1_CIF_ISP_BASE + 0x00000238) +#define RKISP1_CIF_ISP_ERR (RKISP1_CIF_ISP_BASE + 0x0000023C) +#define RKISP1_CIF_ISP_ERR_CLR (RKISP1_CIF_ISP_BASE + 0x00000240) +#define RKISP1_CIF_ISP_FRAME_COUNT (RKISP1_CIF_ISP_BASE + 0x00000244) +#define RKISP1_CIF_ISP_CT_OFFSET_R (RKISP1_CIF_ISP_BASE + 0x00000248) +#define RKISP1_CIF_ISP_CT_OFFSET_G (RKISP1_CIF_ISP_BASE + 0x0000024C) +#define RKISP1_CIF_ISP_CT_OFFSET_B (RKISP1_CIF_ISP_BASE + 0x00000250) + +#define RKISP1_CIF_ISP_FLASH_BASE 0x00000660 +#define RKISP1_CIF_ISP_FLASH_CMD (RKISP1_CIF_ISP_FLASH_BASE + 0x00000000) +#define RKISP1_CIF_ISP_FLASH_CONFIG (RKISP1_CIF_ISP_FLASH_BASE + 0x00000004) +#define RKISP1_CIF_ISP_FLASH_PREDIV (RKISP1_CIF_ISP_FLASH_BASE + 0x00000008) +#define RKISP1_CIF_ISP_FLASH_DELAY (RKISP1_CIF_ISP_FLASH_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_FLASH_TIME (RKISP1_CIF_ISP_FLASH_BASE + 0x00000010) +#define RKISP1_CIF_ISP_FLASH_MAXP (RKISP1_CIF_ISP_FLASH_BASE + 0x00000014) + +#define RKISP1_CIF_ISP_SH_BASE 0x00000680 +#define RKISP1_CIF_ISP_SH_CTRL (RKISP1_CIF_ISP_SH_BASE + 0x00000000) +#define RKISP1_CIF_ISP_SH_PREDIV (RKISP1_CIF_ISP_SH_BASE + 0x00000004) +#define RKISP1_CIF_ISP_SH_DELAY (RKISP1_CIF_ISP_SH_BASE + 0x00000008) +#define RKISP1_CIF_ISP_SH_TIME (RKISP1_CIF_ISP_SH_BASE + 0x0000000C) + +#define RKISP1_CIF_C_PROC_BASE 0x00000800 +#define RKISP1_CIF_C_PROC_CTRL (RKISP1_CIF_C_PROC_BASE + 0x00000000) +#define RKISP1_CIF_C_PROC_CONTRAST (RKISP1_CIF_C_PROC_BASE + 0x00000004) +#define RKISP1_CIF_C_PROC_BRIGHTNESS (RKISP1_CIF_C_PROC_BASE + 0x00000008) +#define RKISP1_CIF_C_PROC_SATURATION (RKISP1_CIF_C_PROC_BASE + 0x0000000C) +#define RKISP1_CIF_C_PROC_HUE (RKISP1_CIF_C_PROC_BASE + 0x00000010) + +#define RKISP1_CIF_DUAL_CROP_BASE 0x00000880 +#define RKISP1_CIF_DUAL_CROP_CTRL (RKISP1_CIF_DUAL_CROP_BASE + 0x00000000) +#define RKISP1_CIF_DUAL_CROP_M_H_OFFS (RKISP1_CIF_DUAL_CROP_BASE + 0x00000004) +#define RKISP1_CIF_DUAL_CROP_M_V_OFFS (RKISP1_CIF_DUAL_CROP_BASE + 0x00000008) +#define RKISP1_CIF_DUAL_CROP_M_H_SIZE (RKISP1_CIF_DUAL_CROP_BASE + 0x0000000C) +#define RKISP1_CIF_DUAL_CROP_M_V_SIZE (RKISP1_CIF_DUAL_CROP_BASE + 0x00000010) +#define RKISP1_CIF_DUAL_CROP_S_H_OFFS (RKISP1_CIF_DUAL_CROP_BASE + 0x00000014) +#define RKISP1_CIF_DUAL_CROP_S_V_OFFS (RKISP1_CIF_DUAL_CROP_BASE + 0x00000018) +#define RKISP1_CIF_DUAL_CROP_S_H_SIZE (RKISP1_CIF_DUAL_CROP_BASE + 0x0000001C) +#define RKISP1_CIF_DUAL_CROP_S_V_SIZE (RKISP1_CIF_DUAL_CROP_BASE + 0x00000020) +#define RKISP1_CIF_DUAL_CROP_M_H_OFFS_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x00000024) +#define RKISP1_CIF_DUAL_CROP_M_V_OFFS_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x00000028) +#define RKISP1_CIF_DUAL_CROP_M_H_SIZE_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x0000002C) +#define RKISP1_CIF_DUAL_CROP_M_V_SIZE_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x00000030) +#define RKISP1_CIF_DUAL_CROP_S_H_OFFS_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x00000034) +#define RKISP1_CIF_DUAL_CROP_S_V_OFFS_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x00000038) +#define RKISP1_CIF_DUAL_CROP_S_H_SIZE_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x0000003C) +#define RKISP1_CIF_DUAL_CROP_S_V_SIZE_SHD (RKISP1_CIF_DUAL_CROP_BASE + 0x00000040) + +#define RKISP1_CIF_MRSZ_BASE 0x00000C00 +#define RKISP1_CIF_MRSZ_CTRL (RKISP1_CIF_MRSZ_BASE + 0x00000000) +#define RKISP1_CIF_MRSZ_SCALE_HY (RKISP1_CIF_MRSZ_BASE + 0x00000004) +#define RKISP1_CIF_MRSZ_SCALE_HCB (RKISP1_CIF_MRSZ_BASE + 0x00000008) +#define RKISP1_CIF_MRSZ_SCALE_HCR (RKISP1_CIF_MRSZ_BASE + 0x0000000C) +#define RKISP1_CIF_MRSZ_SCALE_VY (RKISP1_CIF_MRSZ_BASE + 0x00000010) +#define RKISP1_CIF_MRSZ_SCALE_VC (RKISP1_CIF_MRSZ_BASE + 0x00000014) +#define RKISP1_CIF_MRSZ_PHASE_HY (RKISP1_CIF_MRSZ_BASE + 0x00000018) +#define RKISP1_CIF_MRSZ_PHASE_HC (RKISP1_CIF_MRSZ_BASE + 0x0000001C) +#define RKISP1_CIF_MRSZ_PHASE_VY (RKISP1_CIF_MRSZ_BASE + 0x00000020) +#define RKISP1_CIF_MRSZ_PHASE_VC (RKISP1_CIF_MRSZ_BASE + 0x00000024) +#define RKISP1_CIF_MRSZ_SCALE_LUT_ADDR (RKISP1_CIF_MRSZ_BASE + 0x00000028) +#define RKISP1_CIF_MRSZ_SCALE_LUT (RKISP1_CIF_MRSZ_BASE + 0x0000002C) +#define RKISP1_CIF_MRSZ_CTRL_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000030) +#define RKISP1_CIF_MRSZ_SCALE_HY_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000034) +#define RKISP1_CIF_MRSZ_SCALE_HCB_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000038) +#define RKISP1_CIF_MRSZ_SCALE_HCR_SHD (RKISP1_CIF_MRSZ_BASE + 0x0000003C) +#define RKISP1_CIF_MRSZ_SCALE_VY_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000040) +#define RKISP1_CIF_MRSZ_SCALE_VC_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000044) +#define RKISP1_CIF_MRSZ_PHASE_HY_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000048) +#define RKISP1_CIF_MRSZ_PHASE_HC_SHD (RKISP1_CIF_MRSZ_BASE + 0x0000004C) +#define RKISP1_CIF_MRSZ_PHASE_VY_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000050) +#define RKISP1_CIF_MRSZ_PHASE_VC_SHD (RKISP1_CIF_MRSZ_BASE + 0x00000054) + +#define RKISP1_CIF_SRSZ_BASE 0x00001000 +#define RKISP1_CIF_SRSZ_CTRL (RKISP1_CIF_SRSZ_BASE + 0x00000000) +#define RKISP1_CIF_SRSZ_SCALE_HY (RKISP1_CIF_SRSZ_BASE + 0x00000004) +#define RKISP1_CIF_SRSZ_SCALE_HCB (RKISP1_CIF_SRSZ_BASE + 0x00000008) +#define RKISP1_CIF_SRSZ_SCALE_HCR (RKISP1_CIF_SRSZ_BASE + 0x0000000C) +#define RKISP1_CIF_SRSZ_SCALE_VY (RKISP1_CIF_SRSZ_BASE + 0x00000010) +#define RKISP1_CIF_SRSZ_SCALE_VC (RKISP1_CIF_SRSZ_BASE + 0x00000014) +#define RKISP1_CIF_SRSZ_PHASE_HY (RKISP1_CIF_SRSZ_BASE + 0x00000018) +#define RKISP1_CIF_SRSZ_PHASE_HC (RKISP1_CIF_SRSZ_BASE + 0x0000001C) +#define RKISP1_CIF_SRSZ_PHASE_VY (RKISP1_CIF_SRSZ_BASE + 0x00000020) +#define RKISP1_CIF_SRSZ_PHASE_VC (RKISP1_CIF_SRSZ_BASE + 0x00000024) +#define RKISP1_CIF_SRSZ_SCALE_LUT_ADDR (RKISP1_CIF_SRSZ_BASE + 0x00000028) +#define RKISP1_CIF_SRSZ_SCALE_LUT (RKISP1_CIF_SRSZ_BASE + 0x0000002C) +#define RKISP1_CIF_SRSZ_CTRL_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000030) +#define RKISP1_CIF_SRSZ_SCALE_HY_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000034) +#define RKISP1_CIF_SRSZ_SCALE_HCB_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000038) +#define RKISP1_CIF_SRSZ_SCALE_HCR_SHD (RKISP1_CIF_SRSZ_BASE + 0x0000003C) +#define RKISP1_CIF_SRSZ_SCALE_VY_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000040) +#define RKISP1_CIF_SRSZ_SCALE_VC_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000044) +#define RKISP1_CIF_SRSZ_PHASE_HY_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000048) +#define RKISP1_CIF_SRSZ_PHASE_HC_SHD (RKISP1_CIF_SRSZ_BASE + 0x0000004C) +#define RKISP1_CIF_SRSZ_PHASE_VY_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000050) +#define RKISP1_CIF_SRSZ_PHASE_VC_SHD (RKISP1_CIF_SRSZ_BASE + 0x00000054) + +#define RKISP1_CIF_MI_BASE 0x00001400 +#define RKISP1_CIF_MI_CTRL (RKISP1_CIF_MI_BASE + 0x00000000) +#define RKISP1_CIF_MI_INIT (RKISP1_CIF_MI_BASE + 0x00000004) +#define RKISP1_CIF_MI_MP_Y_BASE_AD_INIT (RKISP1_CIF_MI_BASE + 0x00000008) +#define RKISP1_CIF_MI_MP_Y_SIZE_INIT (RKISP1_CIF_MI_BASE + 0x0000000C) +#define RKISP1_CIF_MI_MP_Y_OFFS_CNT_INIT (RKISP1_CIF_MI_BASE + 0x00000010) +#define RKISP1_CIF_MI_MP_Y_OFFS_CNT_START (RKISP1_CIF_MI_BASE + 0x00000014) +#define RKISP1_CIF_MI_MP_Y_IRQ_OFFS_INIT (RKISP1_CIF_MI_BASE + 0x00000018) +#define RKISP1_CIF_MI_MP_CB_BASE_AD_INIT (RKISP1_CIF_MI_BASE + 0x0000001C) +#define RKISP1_CIF_MI_MP_CB_SIZE_INIT (RKISP1_CIF_MI_BASE + 0x00000020) +#define RKISP1_CIF_MI_MP_CB_OFFS_CNT_INIT (RKISP1_CIF_MI_BASE + 0x00000024) +#define RKISP1_CIF_MI_MP_CB_OFFS_CNT_START (RKISP1_CIF_MI_BASE + 0x00000028) +#define RKISP1_CIF_MI_MP_CR_BASE_AD_INIT (RKISP1_CIF_MI_BASE + 0x0000002C) +#define RKISP1_CIF_MI_MP_CR_SIZE_INIT (RKISP1_CIF_MI_BASE + 0x00000030) +#define RKISP1_CIF_MI_MP_CR_OFFS_CNT_INIT (RKISP1_CIF_MI_BASE + 0x00000034) +#define RKISP1_CIF_MI_MP_CR_OFFS_CNT_START (RKISP1_CIF_MI_BASE + 0x00000038) +#define RKISP1_CIF_MI_SP_Y_BASE_AD_INIT (RKISP1_CIF_MI_BASE + 0x0000003C) +#define RKISP1_CIF_MI_SP_Y_SIZE_INIT (RKISP1_CIF_MI_BASE + 0x00000040) +#define RKISP1_CIF_MI_SP_Y_OFFS_CNT_INIT (RKISP1_CIF_MI_BASE + 0x00000044) +#define RKISP1_CIF_MI_SP_Y_OFFS_CNT_START (RKISP1_CIF_MI_BASE + 0x00000048) +#define RKISP1_CIF_MI_SP_Y_LLENGTH (RKISP1_CIF_MI_BASE + 0x0000004C) +#define RKISP1_CIF_MI_SP_CB_BASE_AD_INIT (RKISP1_CIF_MI_BASE + 0x00000050) +#define RKISP1_CIF_MI_SP_CB_SIZE_INIT (RKISP1_CIF_MI_BASE + 0x00000054) +#define RKISP1_CIF_MI_SP_CB_OFFS_CNT_INIT (RKISP1_CIF_MI_BASE + 0x00000058) +#define RKISP1_CIF_MI_SP_CB_OFFS_CNT_START (RKISP1_CIF_MI_BASE + 0x0000005C) +#define RKISP1_CIF_MI_SP_CR_BASE_AD_INIT (RKISP1_CIF_MI_BASE + 0x00000060) +#define RKISP1_CIF_MI_SP_CR_SIZE_INIT (RKISP1_CIF_MI_BASE + 0x00000064) +#define RKISP1_CIF_MI_SP_CR_OFFS_CNT_INIT (RKISP1_CIF_MI_BASE + 0x00000068) +#define RKISP1_CIF_MI_SP_CR_OFFS_CNT_START (RKISP1_CIF_MI_BASE + 0x0000006C) +#define RKISP1_CIF_MI_BYTE_CNT (RKISP1_CIF_MI_BASE + 0x00000070) +#define RKISP1_CIF_MI_CTRL_SHD (RKISP1_CIF_MI_BASE + 0x00000074) +#define RKISP1_CIF_MI_MP_Y_BASE_AD_SHD (RKISP1_CIF_MI_BASE + 0x00000078) +#define RKISP1_CIF_MI_MP_Y_SIZE_SHD (RKISP1_CIF_MI_BASE + 0x0000007C) +#define RKISP1_CIF_MI_MP_Y_OFFS_CNT_SHD (RKISP1_CIF_MI_BASE + 0x00000080) +#define RKISP1_CIF_MI_MP_Y_IRQ_OFFS_SHD (RKISP1_CIF_MI_BASE + 0x00000084) +#define RKISP1_CIF_MI_MP_CB_BASE_AD_SHD (RKISP1_CIF_MI_BASE + 0x00000088) +#define RKISP1_CIF_MI_MP_CB_SIZE_SHD (RKISP1_CIF_MI_BASE + 0x0000008C) +#define RKISP1_CIF_MI_MP_CB_OFFS_CNT_SHD (RKISP1_CIF_MI_BASE + 0x00000090) +#define RKISP1_CIF_MI_MP_CR_BASE_AD_SHD (RKISP1_CIF_MI_BASE + 0x00000094) +#define RKISP1_CIF_MI_MP_CR_SIZE_SHD (RKISP1_CIF_MI_BASE + 0x00000098) +#define RKISP1_CIF_MI_MP_CR_OFFS_CNT_SHD (RKISP1_CIF_MI_BASE + 0x0000009C) +#define RKISP1_CIF_MI_SP_Y_BASE_AD_SHD (RKISP1_CIF_MI_BASE + 0x000000A0) +#define RKISP1_CIF_MI_SP_Y_SIZE_SHD (RKISP1_CIF_MI_BASE + 0x000000A4) +#define RKISP1_CIF_MI_SP_Y_OFFS_CNT_SHD (RKISP1_CIF_MI_BASE + 0x000000A8) +#define RKISP1_CIF_MI_SP_CB_BASE_AD_SHD (RKISP1_CIF_MI_BASE + 0x000000B0) +#define RKISP1_CIF_MI_SP_CB_SIZE_SHD (RKISP1_CIF_MI_BASE + 0x000000B4) +#define RKISP1_CIF_MI_SP_CB_OFFS_CNT_SHD (RKISP1_CIF_MI_BASE + 0x000000B8) +#define RKISP1_CIF_MI_SP_CR_BASE_AD_SHD (RKISP1_CIF_MI_BASE + 0x000000BC) +#define RKISP1_CIF_MI_SP_CR_SIZE_SHD (RKISP1_CIF_MI_BASE + 0x000000C0) +#define RKISP1_CIF_MI_SP_CR_OFFS_CNT_SHD (RKISP1_CIF_MI_BASE + 0x000000C4) +#define RKISP1_CIF_MI_DMA_Y_PIC_START_AD (RKISP1_CIF_MI_BASE + 0x000000C8) +#define RKISP1_CIF_MI_DMA_Y_PIC_WIDTH (RKISP1_CIF_MI_BASE + 0x000000CC) +#define RKISP1_CIF_MI_DMA_Y_LLENGTH (RKISP1_CIF_MI_BASE + 0x000000D0) +#define RKISP1_CIF_MI_DMA_Y_PIC_SIZE (RKISP1_CIF_MI_BASE + 0x000000D4) +#define RKISP1_CIF_MI_DMA_CB_PIC_START_AD (RKISP1_CIF_MI_BASE + 0x000000D8) +#define RKISP1_CIF_MI_DMA_CR_PIC_START_AD (RKISP1_CIF_MI_BASE + 0x000000E8) +#define RKISP1_CIF_MI_IMSC (RKISP1_CIF_MI_BASE + 0x000000F8) +#define RKISP1_CIF_MI_RIS (RKISP1_CIF_MI_BASE + 0x000000FC) +#define RKISP1_CIF_MI_MIS (RKISP1_CIF_MI_BASE + 0x00000100) +#define RKISP1_CIF_MI_ICR (RKISP1_CIF_MI_BASE + 0x00000104) +#define RKISP1_CIF_MI_ISR (RKISP1_CIF_MI_BASE + 0x00000108) +#define RKISP1_CIF_MI_STATUS (RKISP1_CIF_MI_BASE + 0x0000010C) +#define RKISP1_CIF_MI_STATUS_CLR (RKISP1_CIF_MI_BASE + 0x00000110) +#define RKISP1_CIF_MI_SP_Y_PIC_WIDTH (RKISP1_CIF_MI_BASE + 0x00000114) +#define RKISP1_CIF_MI_SP_Y_PIC_HEIGHT (RKISP1_CIF_MI_BASE + 0x00000118) +#define RKISP1_CIF_MI_SP_Y_PIC_SIZE (RKISP1_CIF_MI_BASE + 0x0000011C) +#define RKISP1_CIF_MI_DMA_CTRL (RKISP1_CIF_MI_BASE + 0x00000120) +#define RKISP1_CIF_MI_DMA_START (RKISP1_CIF_MI_BASE + 0x00000124) +#define RKISP1_CIF_MI_DMA_STATUS (RKISP1_CIF_MI_BASE + 0x00000128) +#define RKISP1_CIF_MI_PIXEL_COUNT (RKISP1_CIF_MI_BASE + 0x0000012C) +#define RKISP1_CIF_MI_MP_Y_BASE_AD_INIT2 (RKISP1_CIF_MI_BASE + 0x00000130) +#define RKISP1_CIF_MI_MP_CB_BASE_AD_INIT2 (RKISP1_CIF_MI_BASE + 0x00000134) +#define RKISP1_CIF_MI_MP_CR_BASE_AD_INIT2 (RKISP1_CIF_MI_BASE + 0x00000138) +#define RKISP1_CIF_MI_SP_Y_BASE_AD_INIT2 (RKISP1_CIF_MI_BASE + 0x0000013C) +#define RKISP1_CIF_MI_SP_CB_BASE_AD_INIT2 (RKISP1_CIF_MI_BASE + 0x00000140) +#define RKISP1_CIF_MI_SP_CR_BASE_AD_INIT2 (RKISP1_CIF_MI_BASE + 0x00000144) +#define RKISP1_CIF_MI_XTD_FORMAT_CTRL (RKISP1_CIF_MI_BASE + 0x00000148) + +#define RKISP1_CIF_SMIA_BASE 0x00001A00 +#define RKISP1_CIF_SMIA_CTRL (RKISP1_CIF_SMIA_BASE + 0x00000000) +#define RKISP1_CIF_SMIA_STATUS (RKISP1_CIF_SMIA_BASE + 0x00000004) +#define RKISP1_CIF_SMIA_IMSC (RKISP1_CIF_SMIA_BASE + 0x00000008) +#define RKISP1_CIF_SMIA_RIS (RKISP1_CIF_SMIA_BASE + 0x0000000C) +#define RKISP1_CIF_SMIA_MIS (RKISP1_CIF_SMIA_BASE + 0x00000010) +#define RKISP1_CIF_SMIA_ICR (RKISP1_CIF_SMIA_BASE + 0x00000014) +#define RKISP1_CIF_SMIA_ISR (RKISP1_CIF_SMIA_BASE + 0x00000018) +#define RKISP1_CIF_SMIA_DATA_FORMAT_SEL (RKISP1_CIF_SMIA_BASE + 0x0000001C) +#define RKISP1_CIF_SMIA_SOF_EMB_DATA_LINES (RKISP1_CIF_SMIA_BASE + 0x00000020) +#define RKISP1_CIF_SMIA_EMB_HSTART (RKISP1_CIF_SMIA_BASE + 0x00000024) +#define RKISP1_CIF_SMIA_EMB_HSIZE (RKISP1_CIF_SMIA_BASE + 0x00000028) +#define RKISP1_CIF_SMIA_EMB_VSTART (RKISP1_CIF_SMIA_BASE + 0x0000002c) +#define RKISP1_CIF_SMIA_NUM_LINES (RKISP1_CIF_SMIA_BASE + 0x00000030) +#define RKISP1_CIF_SMIA_EMB_DATA_FIFO (RKISP1_CIF_SMIA_BASE + 0x00000034) +#define RKISP1_CIF_SMIA_EMB_DATA_WATERMARK (RKISP1_CIF_SMIA_BASE + 0x00000038) + +#define RKISP1_CIF_MIPI_BASE 0x00001C00 +#define RKISP1_CIF_MIPI_CTRL (RKISP1_CIF_MIPI_BASE + 0x00000000) +#define RKISP1_CIF_MIPI_STATUS (RKISP1_CIF_MIPI_BASE + 0x00000004) +#define RKISP1_CIF_MIPI_IMSC (RKISP1_CIF_MIPI_BASE + 0x00000008) +#define RKISP1_CIF_MIPI_RIS (RKISP1_CIF_MIPI_BASE + 0x0000000C) +#define RKISP1_CIF_MIPI_MIS (RKISP1_CIF_MIPI_BASE + 0x00000010) +#define RKISP1_CIF_MIPI_ICR (RKISP1_CIF_MIPI_BASE + 0x00000014) +#define RKISP1_CIF_MIPI_ISR (RKISP1_CIF_MIPI_BASE + 0x00000018) +#define RKISP1_CIF_MIPI_CUR_DATA_ID (RKISP1_CIF_MIPI_BASE + 0x0000001C) +#define RKISP1_CIF_MIPI_IMG_DATA_SEL (RKISP1_CIF_MIPI_BASE + 0x00000020) +#define RKISP1_CIF_MIPI_ADD_DATA_SEL_1 (RKISP1_CIF_MIPI_BASE + 0x00000024) +#define RKISP1_CIF_MIPI_ADD_DATA_SEL_2 (RKISP1_CIF_MIPI_BASE + 0x00000028) +#define RKISP1_CIF_MIPI_ADD_DATA_SEL_3 (RKISP1_CIF_MIPI_BASE + 0x0000002C) +#define RKISP1_CIF_MIPI_ADD_DATA_SEL_4 (RKISP1_CIF_MIPI_BASE + 0x00000030) +#define RKISP1_CIF_MIPI_ADD_DATA_FIFO (RKISP1_CIF_MIPI_BASE + 0x00000034) +#define RKISP1_CIF_MIPI_FIFO_FILL_LEVEL (RKISP1_CIF_MIPI_BASE + 0x00000038) +#define RKISP1_CIF_MIPI_COMPRESSED_MODE (RKISP1_CIF_MIPI_BASE + 0x0000003C) +#define RKISP1_CIF_MIPI_FRAME (RKISP1_CIF_MIPI_BASE + 0x00000040) +#define RKISP1_CIF_MIPI_GEN_SHORT_DT (RKISP1_CIF_MIPI_BASE + 0x00000044) +#define RKISP1_CIF_MIPI_GEN_SHORT_8_9 (RKISP1_CIF_MIPI_BASE + 0x00000048) +#define RKISP1_CIF_MIPI_GEN_SHORT_A_B (RKISP1_CIF_MIPI_BASE + 0x0000004C) +#define RKISP1_CIF_MIPI_GEN_SHORT_C_D (RKISP1_CIF_MIPI_BASE + 0x00000050) +#define RKISP1_CIF_MIPI_GEN_SHORT_E_F (RKISP1_CIF_MIPI_BASE + 0x00000054) + +#define RKISP1_CIF_ISP_AFM_BASE 0x00002000 +#define RKISP1_CIF_ISP_AFM_CTRL (RKISP1_CIF_ISP_AFM_BASE + 0x00000000) +#define RKISP1_CIF_ISP_AFM_LT_A (RKISP1_CIF_ISP_AFM_BASE + 0x00000004) +#define RKISP1_CIF_ISP_AFM_RB_A (RKISP1_CIF_ISP_AFM_BASE + 0x00000008) +#define RKISP1_CIF_ISP_AFM_LT_B (RKISP1_CIF_ISP_AFM_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_AFM_RB_B (RKISP1_CIF_ISP_AFM_BASE + 0x00000010) +#define RKISP1_CIF_ISP_AFM_LT_C (RKISP1_CIF_ISP_AFM_BASE + 0x00000014) +#define RKISP1_CIF_ISP_AFM_RB_C (RKISP1_CIF_ISP_AFM_BASE + 0x00000018) +#define RKISP1_CIF_ISP_AFM_THRES (RKISP1_CIF_ISP_AFM_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_AFM_VAR_SHIFT (RKISP1_CIF_ISP_AFM_BASE + 0x00000020) +#define RKISP1_CIF_ISP_AFM_SUM_A (RKISP1_CIF_ISP_AFM_BASE + 0x00000024) +#define RKISP1_CIF_ISP_AFM_SUM_B (RKISP1_CIF_ISP_AFM_BASE + 0x00000028) +#define RKISP1_CIF_ISP_AFM_SUM_C (RKISP1_CIF_ISP_AFM_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_AFM_LUM_A (RKISP1_CIF_ISP_AFM_BASE + 0x00000030) +#define RKISP1_CIF_ISP_AFM_LUM_B (RKISP1_CIF_ISP_AFM_BASE + 0x00000034) +#define RKISP1_CIF_ISP_AFM_LUM_C (RKISP1_CIF_ISP_AFM_BASE + 0x00000038) + +#define RKISP1_CIF_ISP_LSC_BASE 0x00002200 +#define RKISP1_CIF_ISP_LSC_CTRL (RKISP1_CIF_ISP_LSC_BASE + 0x00000000) +#define RKISP1_CIF_ISP_LSC_R_TABLE_ADDR (RKISP1_CIF_ISP_LSC_BASE + 0x00000004) +#define RKISP1_CIF_ISP_LSC_GR_TABLE_ADDR (RKISP1_CIF_ISP_LSC_BASE + 0x00000008) +#define RKISP1_CIF_ISP_LSC_B_TABLE_ADDR (RKISP1_CIF_ISP_LSC_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_LSC_GB_TABLE_ADDR (RKISP1_CIF_ISP_LSC_BASE + 0x00000010) +#define RKISP1_CIF_ISP_LSC_R_TABLE_DATA (RKISP1_CIF_ISP_LSC_BASE + 0x00000014) +#define RKISP1_CIF_ISP_LSC_GR_TABLE_DATA (RKISP1_CIF_ISP_LSC_BASE + 0x00000018) +#define RKISP1_CIF_ISP_LSC_B_TABLE_DATA (RKISP1_CIF_ISP_LSC_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_LSC_GB_TABLE_DATA (RKISP1_CIF_ISP_LSC_BASE + 0x00000020) +#define RKISP1_CIF_ISP_LSC_XGRAD_01 (RKISP1_CIF_ISP_LSC_BASE + 0x00000024) +#define RKISP1_CIF_ISP_LSC_XGRAD_23 (RKISP1_CIF_ISP_LSC_BASE + 0x00000028) +#define RKISP1_CIF_ISP_LSC_XGRAD_45 (RKISP1_CIF_ISP_LSC_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_LSC_XGRAD_67 (RKISP1_CIF_ISP_LSC_BASE + 0x00000030) +#define RKISP1_CIF_ISP_LSC_YGRAD_01 (RKISP1_CIF_ISP_LSC_BASE + 0x00000034) +#define RKISP1_CIF_ISP_LSC_YGRAD_23 (RKISP1_CIF_ISP_LSC_BASE + 0x00000038) +#define RKISP1_CIF_ISP_LSC_YGRAD_45 (RKISP1_CIF_ISP_LSC_BASE + 0x0000003C) +#define RKISP1_CIF_ISP_LSC_YGRAD_67 (RKISP1_CIF_ISP_LSC_BASE + 0x00000040) +#define RKISP1_CIF_ISP_LSC_XSIZE_01 (RKISP1_CIF_ISP_LSC_BASE + 0x00000044) +#define RKISP1_CIF_ISP_LSC_XSIZE_23 (RKISP1_CIF_ISP_LSC_BASE + 0x00000048) +#define RKISP1_CIF_ISP_LSC_XSIZE_45 (RKISP1_CIF_ISP_LSC_BASE + 0x0000004C) +#define RKISP1_CIF_ISP_LSC_XSIZE_67 (RKISP1_CIF_ISP_LSC_BASE + 0x00000050) +#define RKISP1_CIF_ISP_LSC_YSIZE_01 (RKISP1_CIF_ISP_LSC_BASE + 0x00000054) +#define RKISP1_CIF_ISP_LSC_YSIZE_23 (RKISP1_CIF_ISP_LSC_BASE + 0x00000058) +#define RKISP1_CIF_ISP_LSC_YSIZE_45 (RKISP1_CIF_ISP_LSC_BASE + 0x0000005C) +#define RKISP1_CIF_ISP_LSC_YSIZE_67 (RKISP1_CIF_ISP_LSC_BASE + 0x00000060) +#define RKISP1_CIF_ISP_LSC_TABLE_SEL (RKISP1_CIF_ISP_LSC_BASE + 0x00000064) +#define RKISP1_CIF_ISP_LSC_STATUS (RKISP1_CIF_ISP_LSC_BASE + 0x00000068) + +#define RKISP1_CIF_ISP_IS_BASE 0x00002300 +#define RKISP1_CIF_ISP_IS_CTRL (RKISP1_CIF_ISP_IS_BASE + 0x00000000) +#define RKISP1_CIF_ISP_IS_RECENTER (RKISP1_CIF_ISP_IS_BASE + 0x00000004) +#define RKISP1_CIF_ISP_IS_H_OFFS (RKISP1_CIF_ISP_IS_BASE + 0x00000008) +#define RKISP1_CIF_ISP_IS_V_OFFS (RKISP1_CIF_ISP_IS_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_IS_H_SIZE (RKISP1_CIF_ISP_IS_BASE + 0x00000010) +#define RKISP1_CIF_ISP_IS_V_SIZE (RKISP1_CIF_ISP_IS_BASE + 0x00000014) +#define RKISP1_CIF_ISP_IS_MAX_DX (RKISP1_CIF_ISP_IS_BASE + 0x00000018) +#define RKISP1_CIF_ISP_IS_MAX_DY (RKISP1_CIF_ISP_IS_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_IS_DISPLACE (RKISP1_CIF_ISP_IS_BASE + 0x00000020) +#define RKISP1_CIF_ISP_IS_H_OFFS_SHD (RKISP1_CIF_ISP_IS_BASE + 0x00000024) +#define RKISP1_CIF_ISP_IS_V_OFFS_SHD (RKISP1_CIF_ISP_IS_BASE + 0x00000028) +#define RKISP1_CIF_ISP_IS_H_SIZE_SHD (RKISP1_CIF_ISP_IS_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_IS_V_SIZE_SHD (RKISP1_CIF_ISP_IS_BASE + 0x00000030) + +#define RKISP1_CIF_ISP_HIST_BASE 0x00002400 + +#define RKISP1_CIF_ISP_HIST_PROP (RKISP1_CIF_ISP_HIST_BASE + 0x00000000) +#define RKISP1_CIF_ISP_HIST_H_OFFS (RKISP1_CIF_ISP_HIST_BASE + 0x00000004) +#define RKISP1_CIF_ISP_HIST_V_OFFS (RKISP1_CIF_ISP_HIST_BASE + 0x00000008) +#define RKISP1_CIF_ISP_HIST_H_SIZE (RKISP1_CIF_ISP_HIST_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_HIST_V_SIZE (RKISP1_CIF_ISP_HIST_BASE + 0x00000010) +#define RKISP1_CIF_ISP_HIST_BIN_0 (RKISP1_CIF_ISP_HIST_BASE + 0x00000014) +#define RKISP1_CIF_ISP_HIST_BIN_1 (RKISP1_CIF_ISP_HIST_BASE + 0x00000018) +#define RKISP1_CIF_ISP_HIST_BIN_2 (RKISP1_CIF_ISP_HIST_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_HIST_BIN_3 (RKISP1_CIF_ISP_HIST_BASE + 0x00000020) +#define RKISP1_CIF_ISP_HIST_BIN_4 (RKISP1_CIF_ISP_HIST_BASE + 0x00000024) +#define RKISP1_CIF_ISP_HIST_BIN_5 (RKISP1_CIF_ISP_HIST_BASE + 0x00000028) +#define RKISP1_CIF_ISP_HIST_BIN_6 (RKISP1_CIF_ISP_HIST_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_HIST_BIN_7 (RKISP1_CIF_ISP_HIST_BASE + 0x00000030) +#define RKISP1_CIF_ISP_HIST_BIN_8 (RKISP1_CIF_ISP_HIST_BASE + 0x00000034) +#define RKISP1_CIF_ISP_HIST_BIN_9 (RKISP1_CIF_ISP_HIST_BASE + 0x00000038) +#define RKISP1_CIF_ISP_HIST_BIN_10 (RKISP1_CIF_ISP_HIST_BASE + 0x0000003C) +#define RKISP1_CIF_ISP_HIST_BIN_11 (RKISP1_CIF_ISP_HIST_BASE + 0x00000040) +#define RKISP1_CIF_ISP_HIST_BIN_12 (RKISP1_CIF_ISP_HIST_BASE + 0x00000044) +#define RKISP1_CIF_ISP_HIST_BIN_13 (RKISP1_CIF_ISP_HIST_BASE + 0x00000048) +#define RKISP1_CIF_ISP_HIST_BIN_14 (RKISP1_CIF_ISP_HIST_BASE + 0x0000004C) +#define RKISP1_CIF_ISP_HIST_BIN_15 (RKISP1_CIF_ISP_HIST_BASE + 0x00000050) +#define RKISP1_CIF_ISP_HIST_WEIGHT_00TO30 (RKISP1_CIF_ISP_HIST_BASE + 0x00000054) +#define RKISP1_CIF_ISP_HIST_WEIGHT_40TO21 (RKISP1_CIF_ISP_HIST_BASE + 0x00000058) +#define RKISP1_CIF_ISP_HIST_WEIGHT_31TO12 (RKISP1_CIF_ISP_HIST_BASE + 0x0000005C) +#define RKISP1_CIF_ISP_HIST_WEIGHT_22TO03 (RKISP1_CIF_ISP_HIST_BASE + 0x00000060) +#define RKISP1_CIF_ISP_HIST_WEIGHT_13TO43 (RKISP1_CIF_ISP_HIST_BASE + 0x00000064) +#define RKISP1_CIF_ISP_HIST_WEIGHT_04TO34 (RKISP1_CIF_ISP_HIST_BASE + 0x00000068) +#define RKISP1_CIF_ISP_HIST_WEIGHT_44 (RKISP1_CIF_ISP_HIST_BASE + 0x0000006C) + +#define RKISP1_CIF_ISP_FILT_BASE 0x00002500 +#define RKISP1_CIF_ISP_FILT_MODE (RKISP1_CIF_ISP_FILT_BASE + 0x00000000) +#define RKISP1_CIF_ISP_FILT_THRESH_BL0 (RKISP1_CIF_ISP_FILT_BASE + 0x00000028) +#define RKISP1_CIF_ISP_FILT_THRESH_BL1 (RKISP1_CIF_ISP_FILT_BASE + 0x0000002c) +#define RKISP1_CIF_ISP_FILT_THRESH_SH0 (RKISP1_CIF_ISP_FILT_BASE + 0x00000030) +#define RKISP1_CIF_ISP_FILT_THRESH_SH1 (RKISP1_CIF_ISP_FILT_BASE + 0x00000034) +#define RKISP1_CIF_ISP_FILT_LUM_WEIGHT (RKISP1_CIF_ISP_FILT_BASE + 0x00000038) +#define RKISP1_CIF_ISP_FILT_FAC_SH1 (RKISP1_CIF_ISP_FILT_BASE + 0x0000003c) +#define RKISP1_CIF_ISP_FILT_FAC_SH0 (RKISP1_CIF_ISP_FILT_BASE + 0x00000040) +#define RKISP1_CIF_ISP_FILT_FAC_MID (RKISP1_CIF_ISP_FILT_BASE + 0x00000044) +#define RKISP1_CIF_ISP_FILT_FAC_BL0 (RKISP1_CIF_ISP_FILT_BASE + 0x00000048) +#define RKISP1_CIF_ISP_FILT_FAC_BL1 (RKISP1_CIF_ISP_FILT_BASE + 0x0000004C) + +#define RKISP1_CIF_ISP_CAC_BASE 0x00002580 +#define RKISP1_CIF_ISP_CAC_CTRL (RKISP1_CIF_ISP_CAC_BASE + 0x00000000) +#define RKISP1_CIF_ISP_CAC_COUNT_START (RKISP1_CIF_ISP_CAC_BASE + 0x00000004) +#define RKISP1_CIF_ISP_CAC_A (RKISP1_CIF_ISP_CAC_BASE + 0x00000008) +#define RKISP1_CIF_ISP_CAC_B (RKISP1_CIF_ISP_CAC_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_CAC_C (RKISP1_CIF_ISP_CAC_BASE + 0x00000010) +#define RKISP1_CIF_ISP_X_NORM (RKISP1_CIF_ISP_CAC_BASE + 0x00000014) +#define RKISP1_CIF_ISP_Y_NORM (RKISP1_CIF_ISP_CAC_BASE + 0x00000018) + +#define RKISP1_CIF_ISP_EXP_BASE 0x00002600 +#define RKISP1_CIF_ISP_EXP_CTRL (RKISP1_CIF_ISP_EXP_BASE + 0x00000000) +#define RKISP1_CIF_ISP_EXP_H_OFFSET (RKISP1_CIF_ISP_EXP_BASE + 0x00000004) +#define RKISP1_CIF_ISP_EXP_V_OFFSET (RKISP1_CIF_ISP_EXP_BASE + 0x00000008) +#define RKISP1_CIF_ISP_EXP_H_SIZE (RKISP1_CIF_ISP_EXP_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_EXP_V_SIZE (RKISP1_CIF_ISP_EXP_BASE + 0x00000010) +#define RKISP1_CIF_ISP_EXP_MEAN_00 (RKISP1_CIF_ISP_EXP_BASE + 0x00000014) +#define RKISP1_CIF_ISP_EXP_MEAN_10 (RKISP1_CIF_ISP_EXP_BASE + 0x00000018) +#define RKISP1_CIF_ISP_EXP_MEAN_20 (RKISP1_CIF_ISP_EXP_BASE + 0x0000001c) +#define RKISP1_CIF_ISP_EXP_MEAN_30 (RKISP1_CIF_ISP_EXP_BASE + 0x00000020) +#define RKISP1_CIF_ISP_EXP_MEAN_40 (RKISP1_CIF_ISP_EXP_BASE + 0x00000024) +#define RKISP1_CIF_ISP_EXP_MEAN_01 (RKISP1_CIF_ISP_EXP_BASE + 0x00000028) +#define RKISP1_CIF_ISP_EXP_MEAN_11 (RKISP1_CIF_ISP_EXP_BASE + 0x0000002c) +#define RKISP1_CIF_ISP_EXP_MEAN_21 (RKISP1_CIF_ISP_EXP_BASE + 0x00000030) +#define RKISP1_CIF_ISP_EXP_MEAN_31 (RKISP1_CIF_ISP_EXP_BASE + 0x00000034) +#define RKISP1_CIF_ISP_EXP_MEAN_41 (RKISP1_CIF_ISP_EXP_BASE + 0x00000038) +#define RKISP1_CIF_ISP_EXP_MEAN_02 (RKISP1_CIF_ISP_EXP_BASE + 0x0000003c) +#define RKISP1_CIF_ISP_EXP_MEAN_12 (RKISP1_CIF_ISP_EXP_BASE + 0x00000040) +#define RKISP1_CIF_ISP_EXP_MEAN_22 (RKISP1_CIF_ISP_EXP_BASE + 0x00000044) +#define RKISP1_CIF_ISP_EXP_MEAN_32 (RKISP1_CIF_ISP_EXP_BASE + 0x00000048) +#define RKISP1_CIF_ISP_EXP_MEAN_42 (RKISP1_CIF_ISP_EXP_BASE + 0x0000004c) +#define RKISP1_CIF_ISP_EXP_MEAN_03 (RKISP1_CIF_ISP_EXP_BASE + 0x00000050) +#define RKISP1_CIF_ISP_EXP_MEAN_13 (RKISP1_CIF_ISP_EXP_BASE + 0x00000054) +#define RKISP1_CIF_ISP_EXP_MEAN_23 (RKISP1_CIF_ISP_EXP_BASE + 0x00000058) +#define RKISP1_CIF_ISP_EXP_MEAN_33 (RKISP1_CIF_ISP_EXP_BASE + 0x0000005c) +#define RKISP1_CIF_ISP_EXP_MEAN_43 (RKISP1_CIF_ISP_EXP_BASE + 0x00000060) +#define RKISP1_CIF_ISP_EXP_MEAN_04 (RKISP1_CIF_ISP_EXP_BASE + 0x00000064) +#define RKISP1_CIF_ISP_EXP_MEAN_14 (RKISP1_CIF_ISP_EXP_BASE + 0x00000068) +#define RKISP1_CIF_ISP_EXP_MEAN_24 (RKISP1_CIF_ISP_EXP_BASE + 0x0000006c) +#define RKISP1_CIF_ISP_EXP_MEAN_34 (RKISP1_CIF_ISP_EXP_BASE + 0x00000070) +#define RKISP1_CIF_ISP_EXP_MEAN_44 (RKISP1_CIF_ISP_EXP_BASE + 0x00000074) + +#define RKISP1_CIF_ISP_BLS_BASE 0x00002700 +#define RKISP1_CIF_ISP_BLS_CTRL (RKISP1_CIF_ISP_BLS_BASE + 0x00000000) +#define RKISP1_CIF_ISP_BLS_SAMPLES (RKISP1_CIF_ISP_BLS_BASE + 0x00000004) +#define RKISP1_CIF_ISP_BLS_H1_START (RKISP1_CIF_ISP_BLS_BASE + 0x00000008) +#define RKISP1_CIF_ISP_BLS_H1_STOP (RKISP1_CIF_ISP_BLS_BASE + 0x0000000c) +#define RKISP1_CIF_ISP_BLS_V1_START (RKISP1_CIF_ISP_BLS_BASE + 0x00000010) +#define RKISP1_CIF_ISP_BLS_V1_STOP (RKISP1_CIF_ISP_BLS_BASE + 0x00000014) +#define RKISP1_CIF_ISP_BLS_H2_START (RKISP1_CIF_ISP_BLS_BASE + 0x00000018) +#define RKISP1_CIF_ISP_BLS_H2_STOP (RKISP1_CIF_ISP_BLS_BASE + 0x0000001c) +#define RKISP1_CIF_ISP_BLS_V2_START (RKISP1_CIF_ISP_BLS_BASE + 0x00000020) +#define RKISP1_CIF_ISP_BLS_V2_STOP (RKISP1_CIF_ISP_BLS_BASE + 0x00000024) +#define RKISP1_CIF_ISP_BLS_A_FIXED (RKISP1_CIF_ISP_BLS_BASE + 0x00000028) +#define RKISP1_CIF_ISP_BLS_B_FIXED (RKISP1_CIF_ISP_BLS_BASE + 0x0000002c) +#define RKISP1_CIF_ISP_BLS_C_FIXED (RKISP1_CIF_ISP_BLS_BASE + 0x00000030) +#define RKISP1_CIF_ISP_BLS_D_FIXED (RKISP1_CIF_ISP_BLS_BASE + 0x00000034) +#define RKISP1_CIF_ISP_BLS_A_MEASURED (RKISP1_CIF_ISP_BLS_BASE + 0x00000038) +#define RKISP1_CIF_ISP_BLS_B_MEASURED (RKISP1_CIF_ISP_BLS_BASE + 0x0000003c) +#define RKISP1_CIF_ISP_BLS_C_MEASURED (RKISP1_CIF_ISP_BLS_BASE + 0x00000040) +#define RKISP1_CIF_ISP_BLS_D_MEASURED (RKISP1_CIF_ISP_BLS_BASE + 0x00000044) + +#define RKISP1_CIF_ISP_DPF_BASE 0x00002800 +#define RKISP1_CIF_ISP_DPF_MODE (RKISP1_CIF_ISP_DPF_BASE + 0x00000000) +#define RKISP1_CIF_ISP_DPF_STRENGTH_R (RKISP1_CIF_ISP_DPF_BASE + 0x00000004) +#define RKISP1_CIF_ISP_DPF_STRENGTH_G (RKISP1_CIF_ISP_DPF_BASE + 0x00000008) +#define RKISP1_CIF_ISP_DPF_STRENGTH_B (RKISP1_CIF_ISP_DPF_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_DPF_S_WEIGHT_G_1_4 (RKISP1_CIF_ISP_DPF_BASE + 0x00000010) +#define RKISP1_CIF_ISP_DPF_S_WEIGHT_G_5_6 (RKISP1_CIF_ISP_DPF_BASE + 0x00000014) +#define RKISP1_CIF_ISP_DPF_S_WEIGHT_RB_1_4 (RKISP1_CIF_ISP_DPF_BASE + 0x00000018) +#define RKISP1_CIF_ISP_DPF_S_WEIGHT_RB_5_6 (RKISP1_CIF_ISP_DPF_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_0 (RKISP1_CIF_ISP_DPF_BASE + 0x00000020) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_1 (RKISP1_CIF_ISP_DPF_BASE + 0x00000024) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_2 (RKISP1_CIF_ISP_DPF_BASE + 0x00000028) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_3 (RKISP1_CIF_ISP_DPF_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_4 (RKISP1_CIF_ISP_DPF_BASE + 0x00000030) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_5 (RKISP1_CIF_ISP_DPF_BASE + 0x00000034) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_6 (RKISP1_CIF_ISP_DPF_BASE + 0x00000038) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_7 (RKISP1_CIF_ISP_DPF_BASE + 0x0000003C) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_8 (RKISP1_CIF_ISP_DPF_BASE + 0x00000040) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_9 (RKISP1_CIF_ISP_DPF_BASE + 0x00000044) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_10 (RKISP1_CIF_ISP_DPF_BASE + 0x00000048) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_11 (RKISP1_CIF_ISP_DPF_BASE + 0x0000004C) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_12 (RKISP1_CIF_ISP_DPF_BASE + 0x00000050) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_13 (RKISP1_CIF_ISP_DPF_BASE + 0x00000054) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_14 (RKISP1_CIF_ISP_DPF_BASE + 0x00000058) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_15 (RKISP1_CIF_ISP_DPF_BASE + 0x0000005C) +#define RKISP1_CIF_ISP_DPF_NULL_COEFF_16 (RKISP1_CIF_ISP_DPF_BASE + 0x00000060) +#define RKISP1_CIF_ISP_DPF_NF_GAIN_R (RKISP1_CIF_ISP_DPF_BASE + 0x00000064) +#define RKISP1_CIF_ISP_DPF_NF_GAIN_GR (RKISP1_CIF_ISP_DPF_BASE + 0x00000068) +#define RKISP1_CIF_ISP_DPF_NF_GAIN_GB (RKISP1_CIF_ISP_DPF_BASE + 0x0000006C) +#define RKISP1_CIF_ISP_DPF_NF_GAIN_B (RKISP1_CIF_ISP_DPF_BASE + 0x00000070) + +#define RKISP1_CIF_ISP_DPCC_BASE 0x00002900 +#define RKISP1_CIF_ISP_DPCC_MODE (RKISP1_CIF_ISP_DPCC_BASE + 0x00000000) +#define RKISP1_CIF_ISP_DPCC_OUTPUT_MODE (RKISP1_CIF_ISP_DPCC_BASE + 0x00000004) +#define RKISP1_CIF_ISP_DPCC_SET_USE (RKISP1_CIF_ISP_DPCC_BASE + 0x00000008) +#define RKISP1_CIF_ISP_DPCC_METHODS_SET_1 (RKISP1_CIF_ISP_DPCC_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_DPCC_METHODS_SET_2 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000010) +#define RKISP1_CIF_ISP_DPCC_METHODS_SET_3 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000014) +#define RKISP1_CIF_ISP_DPCC_LINE_THRESH_1 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000018) +#define RKISP1_CIF_ISP_DPCC_LINE_MAD_FAC_1 (RKISP1_CIF_ISP_DPCC_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_DPCC_PG_FAC_1 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000020) +#define RKISP1_CIF_ISP_DPCC_RND_THRESH_1 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000024) +#define RKISP1_CIF_ISP_DPCC_RG_FAC_1 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000028) +#define RKISP1_CIF_ISP_DPCC_LINE_THRESH_2 (RKISP1_CIF_ISP_DPCC_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_DPCC_LINE_MAD_FAC_2 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000030) +#define RKISP1_CIF_ISP_DPCC_PG_FAC_2 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000034) +#define RKISP1_CIF_ISP_DPCC_RND_THRESH_2 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000038) +#define RKISP1_CIF_ISP_DPCC_RG_FAC_2 (RKISP1_CIF_ISP_DPCC_BASE + 0x0000003C) +#define RKISP1_CIF_ISP_DPCC_LINE_THRESH_3 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000040) +#define RKISP1_CIF_ISP_DPCC_LINE_MAD_FAC_3 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000044) +#define RKISP1_CIF_ISP_DPCC_PG_FAC_3 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000048) +#define RKISP1_CIF_ISP_DPCC_RND_THRESH_3 (RKISP1_CIF_ISP_DPCC_BASE + 0x0000004C) +#define RKISP1_CIF_ISP_DPCC_RG_FAC_3 (RKISP1_CIF_ISP_DPCC_BASE + 0x00000050) +#define RKISP1_CIF_ISP_DPCC_RO_LIMITS (RKISP1_CIF_ISP_DPCC_BASE + 0x00000054) +#define RKISP1_CIF_ISP_DPCC_RND_OFFS (RKISP1_CIF_ISP_DPCC_BASE + 0x00000058) +#define RKISP1_CIF_ISP_DPCC_BPT_CTRL (RKISP1_CIF_ISP_DPCC_BASE + 0x0000005C) +#define RKISP1_CIF_ISP_DPCC_BPT_NUMBER (RKISP1_CIF_ISP_DPCC_BASE + 0x00000060) +#define RKISP1_CIF_ISP_DPCC_BPT_ADDR (RKISP1_CIF_ISP_DPCC_BASE + 0x00000064) +#define RKISP1_CIF_ISP_DPCC_BPT_DATA (RKISP1_CIF_ISP_DPCC_BASE + 0x00000068) + +#define RKISP1_CIF_ISP_WDR_BASE 0x00002A00 +#define RKISP1_CIF_ISP_WDR_CTRL (RKISP1_CIF_ISP_WDR_BASE + 0x00000000) +#define RKISP1_CIF_ISP_WDR_TONECURVE_1 (RKISP1_CIF_ISP_WDR_BASE + 0x00000004) +#define RKISP1_CIF_ISP_WDR_TONECURVE_2 (RKISP1_CIF_ISP_WDR_BASE + 0x00000008) +#define RKISP1_CIF_ISP_WDR_TONECURVE_3 (RKISP1_CIF_ISP_WDR_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_4 (RKISP1_CIF_ISP_WDR_BASE + 0x00000010) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_0 (RKISP1_CIF_ISP_WDR_BASE + 0x00000014) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_1 (RKISP1_CIF_ISP_WDR_BASE + 0x00000018) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_2 (RKISP1_CIF_ISP_WDR_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_3 (RKISP1_CIF_ISP_WDR_BASE + 0x00000020) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_4 (RKISP1_CIF_ISP_WDR_BASE + 0x00000024) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_5 (RKISP1_CIF_ISP_WDR_BASE + 0x00000028) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_6 (RKISP1_CIF_ISP_WDR_BASE + 0x0000002C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_7 (RKISP1_CIF_ISP_WDR_BASE + 0x00000030) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_8 (RKISP1_CIF_ISP_WDR_BASE + 0x00000034) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_9 (RKISP1_CIF_ISP_WDR_BASE + 0x00000038) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_10 (RKISP1_CIF_ISP_WDR_BASE + 0x0000003C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_11 (RKISP1_CIF_ISP_WDR_BASE + 0x00000040) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_12 (RKISP1_CIF_ISP_WDR_BASE + 0x00000044) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_13 (RKISP1_CIF_ISP_WDR_BASE + 0x00000048) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_14 (RKISP1_CIF_ISP_WDR_BASE + 0x0000004C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_15 (RKISP1_CIF_ISP_WDR_BASE + 0x00000050) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_16 (RKISP1_CIF_ISP_WDR_BASE + 0x00000054) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_17 (RKISP1_CIF_ISP_WDR_BASE + 0x00000058) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_18 (RKISP1_CIF_ISP_WDR_BASE + 0x0000005C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_19 (RKISP1_CIF_ISP_WDR_BASE + 0x00000060) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_20 (RKISP1_CIF_ISP_WDR_BASE + 0x00000064) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_21 (RKISP1_CIF_ISP_WDR_BASE + 0x00000068) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_22 (RKISP1_CIF_ISP_WDR_BASE + 0x0000006C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_23 (RKISP1_CIF_ISP_WDR_BASE + 0x00000070) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_24 (RKISP1_CIF_ISP_WDR_BASE + 0x00000074) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_25 (RKISP1_CIF_ISP_WDR_BASE + 0x00000078) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_26 (RKISP1_CIF_ISP_WDR_BASE + 0x0000007C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_27 (RKISP1_CIF_ISP_WDR_BASE + 0x00000080) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_28 (RKISP1_CIF_ISP_WDR_BASE + 0x00000084) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_29 (RKISP1_CIF_ISP_WDR_BASE + 0x00000088) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_30 (RKISP1_CIF_ISP_WDR_BASE + 0x0000008C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_31 (RKISP1_CIF_ISP_WDR_BASE + 0x00000090) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_32 (RKISP1_CIF_ISP_WDR_BASE + 0x00000094) +#define RKISP1_CIF_ISP_WDR_OFFSET (RKISP1_CIF_ISP_WDR_BASE + 0x00000098) +#define RKISP1_CIF_ISP_WDR_DELTAMIN (RKISP1_CIF_ISP_WDR_BASE + 0x0000009C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_1_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000A0) +#define RKISP1_CIF_ISP_WDR_TONECURVE_2_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000A4) +#define RKISP1_CIF_ISP_WDR_TONECURVE_3_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000A8) +#define RKISP1_CIF_ISP_WDR_TONECURVE_4_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000AC) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_0_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000B0) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_1_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000B4) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_2_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000B8) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_3_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000BC) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_4_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000C0) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_5_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000C4) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_6_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000C8) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_7_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000CC) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_8_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000D0) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_9_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000D4) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_10_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000D8) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_11_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000DC) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_12_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000E0) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_13_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000E4) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_14_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000E8) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_15_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000EC) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_16_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000F0) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_17_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000F4) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_18_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000F8) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_19_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x000000FC) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_20_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000100) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_21_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000104) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_22_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000108) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_23_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x0000010C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_24_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000110) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_25_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000114) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_26_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000118) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_27_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x0000011C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_28_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000120) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_29_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000124) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_30_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000128) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_31_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x0000012C) +#define RKISP1_CIF_ISP_WDR_TONECURVE_YM_32_SHD (RKISP1_CIF_ISP_WDR_BASE + 0x00000130) + +#define RKISP1_CIF_ISP_VSM_BASE 0x00002F00 +#define RKISP1_CIF_ISP_VSM_MODE (RKISP1_CIF_ISP_VSM_BASE + 0x00000000) +#define RKISP1_CIF_ISP_VSM_H_OFFS (RKISP1_CIF_ISP_VSM_BASE + 0x00000004) +#define RKISP1_CIF_ISP_VSM_V_OFFS (RKISP1_CIF_ISP_VSM_BASE + 0x00000008) +#define RKISP1_CIF_ISP_VSM_H_SIZE (RKISP1_CIF_ISP_VSM_BASE + 0x0000000C) +#define RKISP1_CIF_ISP_VSM_V_SIZE (RKISP1_CIF_ISP_VSM_BASE + 0x00000010) +#define RKISP1_CIF_ISP_VSM_H_SEGMENTS (RKISP1_CIF_ISP_VSM_BASE + 0x00000014) +#define RKISP1_CIF_ISP_VSM_V_SEGMENTS (RKISP1_CIF_ISP_VSM_BASE + 0x00000018) +#define RKISP1_CIF_ISP_VSM_DELTA_H (RKISP1_CIF_ISP_VSM_BASE + 0x0000001C) +#define RKISP1_CIF_ISP_VSM_DELTA_V (RKISP1_CIF_ISP_VSM_BASE + 0x00000020) + +#endif /* _RKISP1_REGS_H */ diff --git a/drivers/staging/media/rkisp1/rkisp1-resizer.c b/drivers/staging/media/rkisp1/rkisp1-resizer.c new file mode 100644 index 000000000000..8cdc29c1a178 --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-resizer.c @@ -0,0 +1,775 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip ISP1 Driver - V4l resizer device + * + * Copyright (C) 2019 Collabora, Ltd. + * + * Based on Rockchip ISP1 driver by Rockchip Electronics Co., Ltd. + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#include "rkisp1-common.h" + +#define RKISP1_RSZ_SP_DEV_NAME RKISP1_DRIVER_NAME "_resizer_selfpath" +#define RKISP1_RSZ_MP_DEV_NAME RKISP1_DRIVER_NAME "_resizer_mainpath" + +#define RKISP1_DEF_FMT MEDIA_BUS_FMT_YUYV8_2X8 +#define RKISP1_DEF_FMT_TYPE RKISP1_FMT_YUV + +#define RKISP1_MBUS_FMT_HDIV 2 +#define RKISP1_MBUS_FMT_VDIV 1 + +enum rkisp1_shadow_regs_when { + RKISP1_SHADOW_REGS_SYNC, + RKISP1_SHADOW_REGS_ASYNC, +}; + +struct rkisp1_rsz_config { + /* constrains */ + const int max_rsz_width; + const int max_rsz_height; + const int min_rsz_width; + const int min_rsz_height; + /* registers */ + struct { + u32 ctrl; + u32 ctrl_shd; + u32 scale_hy; + u32 scale_hcr; + u32 scale_hcb; + u32 scale_vy; + u32 scale_vc; + u32 scale_lut; + u32 scale_lut_addr; + u32 scale_hy_shd; + u32 scale_hcr_shd; + u32 scale_hcb_shd; + u32 scale_vy_shd; + u32 scale_vc_shd; + u32 phase_hy; + u32 phase_hc; + u32 phase_vy; + u32 phase_vc; + u32 phase_hy_shd; + u32 phase_hc_shd; + u32 phase_vy_shd; + u32 phase_vc_shd; + } rsz; + struct { + u32 ctrl; + u32 yuvmode_mask; + u32 rawmode_mask; + u32 h_offset; + u32 v_offset; + u32 h_size; + u32 v_size; + } dual_crop; +}; + +static const struct rkisp1_rsz_config rkisp1_rsz_config_mp = { + /* constraints */ + .max_rsz_width = RKISP1_RSZ_MP_SRC_MAX_WIDTH, + .max_rsz_height = RKISP1_RSZ_MP_SRC_MAX_HEIGHT, + .min_rsz_width = RKISP1_RSZ_SRC_MIN_WIDTH, + .min_rsz_height = RKISP1_RSZ_SRC_MIN_HEIGHT, + /* registers */ + .rsz = { + .ctrl = RKISP1_CIF_MRSZ_CTRL, + .scale_hy = RKISP1_CIF_MRSZ_SCALE_HY, + .scale_hcr = RKISP1_CIF_MRSZ_SCALE_HCR, + .scale_hcb = RKISP1_CIF_MRSZ_SCALE_HCB, + .scale_vy = RKISP1_CIF_MRSZ_SCALE_VY, + .scale_vc = RKISP1_CIF_MRSZ_SCALE_VC, + .scale_lut = RKISP1_CIF_MRSZ_SCALE_LUT, + .scale_lut_addr = RKISP1_CIF_MRSZ_SCALE_LUT_ADDR, + .scale_hy_shd = RKISP1_CIF_MRSZ_SCALE_HY_SHD, + .scale_hcr_shd = RKISP1_CIF_MRSZ_SCALE_HCR_SHD, + .scale_hcb_shd = RKISP1_CIF_MRSZ_SCALE_HCB_SHD, + .scale_vy_shd = RKISP1_CIF_MRSZ_SCALE_VY_SHD, + .scale_vc_shd = RKISP1_CIF_MRSZ_SCALE_VC_SHD, + .phase_hy = RKISP1_CIF_MRSZ_PHASE_HY, + .phase_hc = RKISP1_CIF_MRSZ_PHASE_HC, + .phase_vy = RKISP1_CIF_MRSZ_PHASE_VY, + .phase_vc = RKISP1_CIF_MRSZ_PHASE_VC, + .ctrl_shd = RKISP1_CIF_MRSZ_CTRL_SHD, + .phase_hy_shd = RKISP1_CIF_MRSZ_PHASE_HY_SHD, + .phase_hc_shd = RKISP1_CIF_MRSZ_PHASE_HC_SHD, + .phase_vy_shd = RKISP1_CIF_MRSZ_PHASE_VY_SHD, + .phase_vc_shd = RKISP1_CIF_MRSZ_PHASE_VC_SHD, + }, + .dual_crop = { + .ctrl = RKISP1_CIF_DUAL_CROP_CTRL, + .yuvmode_mask = RKISP1_CIF_DUAL_CROP_MP_MODE_YUV, + .rawmode_mask = RKISP1_CIF_DUAL_CROP_MP_MODE_RAW, + .h_offset = RKISP1_CIF_DUAL_CROP_M_H_OFFS, + .v_offset = RKISP1_CIF_DUAL_CROP_M_V_OFFS, + .h_size = RKISP1_CIF_DUAL_CROP_M_H_SIZE, + .v_size = RKISP1_CIF_DUAL_CROP_M_V_SIZE, + }, +}; + +static const struct rkisp1_rsz_config rkisp1_rsz_config_sp = { + /* constraints */ + .max_rsz_width = RKISP1_RSZ_SP_SRC_MAX_WIDTH, + .max_rsz_height = RKISP1_RSZ_SP_SRC_MAX_HEIGHT, + .min_rsz_width = RKISP1_RSZ_SRC_MIN_WIDTH, + .min_rsz_height = RKISP1_RSZ_SRC_MIN_HEIGHT, + /* registers */ + .rsz = { + .ctrl = RKISP1_CIF_SRSZ_CTRL, + .scale_hy = RKISP1_CIF_SRSZ_SCALE_HY, + .scale_hcr = RKISP1_CIF_SRSZ_SCALE_HCR, + .scale_hcb = RKISP1_CIF_SRSZ_SCALE_HCB, + .scale_vy = RKISP1_CIF_SRSZ_SCALE_VY, + .scale_vc = RKISP1_CIF_SRSZ_SCALE_VC, + .scale_lut = RKISP1_CIF_SRSZ_SCALE_LUT, + .scale_lut_addr = RKISP1_CIF_SRSZ_SCALE_LUT_ADDR, + .scale_hy_shd = RKISP1_CIF_SRSZ_SCALE_HY_SHD, + .scale_hcr_shd = RKISP1_CIF_SRSZ_SCALE_HCR_SHD, + .scale_hcb_shd = RKISP1_CIF_SRSZ_SCALE_HCB_SHD, + .scale_vy_shd = RKISP1_CIF_SRSZ_SCALE_VY_SHD, + .scale_vc_shd = RKISP1_CIF_SRSZ_SCALE_VC_SHD, + .phase_hy = RKISP1_CIF_SRSZ_PHASE_HY, + .phase_hc = RKISP1_CIF_SRSZ_PHASE_HC, + .phase_vy = RKISP1_CIF_SRSZ_PHASE_VY, + .phase_vc = RKISP1_CIF_SRSZ_PHASE_VC, + .ctrl_shd = RKISP1_CIF_SRSZ_CTRL_SHD, + .phase_hy_shd = RKISP1_CIF_SRSZ_PHASE_HY_SHD, + .phase_hc_shd = RKISP1_CIF_SRSZ_PHASE_HC_SHD, + .phase_vy_shd = RKISP1_CIF_SRSZ_PHASE_VY_SHD, + .phase_vc_shd = RKISP1_CIF_SRSZ_PHASE_VC_SHD, + }, + .dual_crop = { + .ctrl = RKISP1_CIF_DUAL_CROP_CTRL, + .yuvmode_mask = RKISP1_CIF_DUAL_CROP_SP_MODE_YUV, + .rawmode_mask = RKISP1_CIF_DUAL_CROP_SP_MODE_RAW, + .h_offset = RKISP1_CIF_DUAL_CROP_S_H_OFFS, + .v_offset = RKISP1_CIF_DUAL_CROP_S_V_OFFS, + .h_size = RKISP1_CIF_DUAL_CROP_S_H_SIZE, + .v_size = RKISP1_CIF_DUAL_CROP_S_V_SIZE, + }, +}; + +static struct v4l2_mbus_framefmt * +rkisp1_rsz_get_pad_fmt(struct rkisp1_resizer *rsz, + struct v4l2_subdev_pad_config *cfg, + unsigned int pad, u32 which) +{ + if (which == V4L2_SUBDEV_FORMAT_TRY) + return v4l2_subdev_get_try_format(&rsz->sd, cfg, pad); + else + return v4l2_subdev_get_try_format(&rsz->sd, rsz->pad_cfg, pad); +} + +static struct v4l2_rect * +rkisp1_rsz_get_pad_crop(struct rkisp1_resizer *rsz, + struct v4l2_subdev_pad_config *cfg, + unsigned int pad, u32 which) +{ + if (which == V4L2_SUBDEV_FORMAT_TRY) + return v4l2_subdev_get_try_crop(&rsz->sd, cfg, pad); + else + return v4l2_subdev_get_try_crop(&rsz->sd, rsz->pad_cfg, pad); +} + +/* ---------------------------------------------------------------------------- + * Dual crop hw configs + */ + +static void rkisp1_dcrop_disable(struct rkisp1_resizer *rsz, + enum rkisp1_shadow_regs_when when) +{ + u32 dc_ctrl = rkisp1_read(rsz->rkisp1, rsz->config->dual_crop.ctrl); + u32 mask = ~(rsz->config->dual_crop.yuvmode_mask | + rsz->config->dual_crop.rawmode_mask); + + dc_ctrl &= mask; + if (when == RKISP1_SHADOW_REGS_ASYNC) + dc_ctrl |= RKISP1_CIF_DUAL_CROP_GEN_CFG_UPD; + else + dc_ctrl |= RKISP1_CIF_DUAL_CROP_CFG_UPD; + rkisp1_write(rsz->rkisp1, dc_ctrl, rsz->config->dual_crop.ctrl); +} + +/* configure dual-crop unit */ +static void rkisp1_dcrop_config(struct rkisp1_resizer *rsz) +{ + struct rkisp1_device *rkisp1 = rsz->rkisp1; + struct v4l2_mbus_framefmt *sink_fmt; + struct v4l2_rect *sink_crop; + u32 dc_ctrl; + + sink_crop = rkisp1_rsz_get_pad_crop(rsz, NULL, RKISP1_RSZ_PAD_SINK, + V4L2_SUBDEV_FORMAT_ACTIVE); + sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, NULL, RKISP1_RSZ_PAD_SINK, + V4L2_SUBDEV_FORMAT_ACTIVE); + + if (sink_crop->width == sink_fmt->width && + sink_crop->height == sink_fmt->height && + sink_crop->left == 0 && sink_crop->top == 0) { + rkisp1_dcrop_disable(rsz, RKISP1_SHADOW_REGS_SYNC); + dev_dbg(rkisp1->dev, "capture %d crop disabled\n", rsz->id); + return; + } + + dc_ctrl = rkisp1_read(rkisp1, rsz->config->dual_crop.ctrl); + rkisp1_write(rkisp1, sink_crop->left, rsz->config->dual_crop.h_offset); + rkisp1_write(rkisp1, sink_crop->top, rsz->config->dual_crop.v_offset); + rkisp1_write(rkisp1, sink_crop->width, rsz->config->dual_crop.h_size); + rkisp1_write(rkisp1, sink_crop->height, rsz->config->dual_crop.v_size); + dc_ctrl |= rsz->config->dual_crop.yuvmode_mask; + dc_ctrl |= RKISP1_CIF_DUAL_CROP_CFG_UPD; + rkisp1_write(rkisp1, dc_ctrl, rsz->config->dual_crop.ctrl); + + dev_dbg(rkisp1->dev, "stream %d crop: %dx%d -> %dx%d\n", rsz->id, + sink_fmt->width, sink_fmt->height, + sink_crop->width, sink_crop->height); +} + +/* ---------------------------------------------------------------------------- + * Resizer hw configs + */ + +static void rkisp1_rsz_dump_regs(struct rkisp1_resizer *rsz) +{ + dev_dbg(rsz->rkisp1->dev, + "RSZ_CTRL 0x%08x/0x%08x\n" + "RSZ_SCALE_HY %d/%d\n" + "RSZ_SCALE_HCB %d/%d\n" + "RSZ_SCALE_HCR %d/%d\n" + "RSZ_SCALE_VY %d/%d\n" + "RSZ_SCALE_VC %d/%d\n" + "RSZ_PHASE_HY %d/%d\n" + "RSZ_PHASE_HC %d/%d\n" + "RSZ_PHASE_VY %d/%d\n" + "RSZ_PHASE_VC %d/%d\n", + rkisp1_read(rsz->rkisp1, rsz->config->rsz.ctrl), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.ctrl_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_hy), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_hy_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_hcb), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_hcb_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_hcr), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_hcr_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_vy), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_vy_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_vc), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.scale_vc_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_hy), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_hy_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_hc), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_hc_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_vy), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_vy_shd), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_vc), + rkisp1_read(rsz->rkisp1, rsz->config->rsz.phase_vc_shd)); +} + +static void rkisp1_rsz_update_shadow(struct rkisp1_resizer *rsz, + enum rkisp1_shadow_regs_when when) +{ + u32 ctrl_cfg = rkisp1_read(rsz->rkisp1, rsz->config->rsz.ctrl); + + if (when == RKISP1_SHADOW_REGS_ASYNC) + ctrl_cfg |= RKISP1_CIF_RSZ_CTRL_CFG_UPD_AUTO; + else + ctrl_cfg |= RKISP1_CIF_RSZ_CTRL_CFG_UPD; + + rkisp1_write(rsz->rkisp1, ctrl_cfg, rsz->config->rsz.ctrl); +} + +static u32 rkisp1_rsz_calc_ratio(u32 len_sink, u32 len_src) +{ + if (len_sink < len_src) + return ((len_sink - 1) * RKISP1_CIF_RSZ_SCALER_FACTOR) / + (len_src - 1); + + return ((len_src - 1) * RKISP1_CIF_RSZ_SCALER_FACTOR) / + (len_sink - 1) + 1; +} + +static void rkisp1_rsz_disable(struct rkisp1_resizer *rsz, + enum rkisp1_shadow_regs_when when) +{ + rkisp1_write(rsz->rkisp1, 0, rsz->config->rsz.ctrl); + + if (when == RKISP1_SHADOW_REGS_SYNC) + rkisp1_rsz_update_shadow(rsz, when); +} + +static void rkisp1_rsz_config_regs(struct rkisp1_resizer *rsz, + struct v4l2_rect *sink_y, + struct v4l2_rect *sink_c, + struct v4l2_rect *src_y, + struct v4l2_rect *src_c, + enum rkisp1_shadow_regs_when when) +{ + struct rkisp1_device *rkisp1 = rsz->rkisp1; + u32 ratio, rsz_ctrl = 0; + unsigned int i; + + /* No phase offset */ + rkisp1_write(rkisp1, 0, rsz->config->rsz.phase_hy); + rkisp1_write(rkisp1, 0, rsz->config->rsz.phase_hc); + rkisp1_write(rkisp1, 0, rsz->config->rsz.phase_vy); + rkisp1_write(rkisp1, 0, rsz->config->rsz.phase_vc); + + /* Linear interpolation */ + for (i = 0; i < 64; i++) { + rkisp1_write(rkisp1, i, rsz->config->rsz.scale_lut_addr); + rkisp1_write(rkisp1, i, rsz->config->rsz.scale_lut); + } + + if (sink_y->width != src_y->width) { + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_HY_ENABLE; + if (sink_y->width < src_y->width) + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_HY_UP; + ratio = rkisp1_rsz_calc_ratio(sink_y->width, src_y->width); + rkisp1_write(rkisp1, ratio, rsz->config->rsz.scale_hy); + } + + if (sink_c->width != src_c->width) { + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_HC_ENABLE; + if (sink_c->width < src_c->width) + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_HC_UP; + ratio = rkisp1_rsz_calc_ratio(sink_c->width, src_c->width); + rkisp1_write(rkisp1, ratio, rsz->config->rsz.scale_hcb); + rkisp1_write(rkisp1, ratio, rsz->config->rsz.scale_hcr); + } + + if (sink_y->height != src_y->height) { + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_VY_ENABLE; + if (sink_y->height < src_y->height) + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_VY_UP; + ratio = rkisp1_rsz_calc_ratio(sink_y->height, src_y->height); + rkisp1_write(rkisp1, ratio, rsz->config->rsz.scale_vy); + } + + if (sink_c->height != src_c->height) { + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_VC_ENABLE; + if (sink_c->height < src_c->height) + rsz_ctrl |= RKISP1_CIF_RSZ_CTRL_SCALE_VC_UP; + ratio = rkisp1_rsz_calc_ratio(sink_c->height, src_c->height); + rkisp1_write(rkisp1, ratio, rsz->config->rsz.scale_vc); + } + + rkisp1_write(rkisp1, rsz_ctrl, rsz->config->rsz.ctrl); + + rkisp1_rsz_update_shadow(rsz, when); +} + +static void rkisp1_rsz_config(struct rkisp1_resizer *rsz, + enum rkisp1_shadow_regs_when when) +{ + u8 hdiv = RKISP1_MBUS_FMT_HDIV, vdiv = RKISP1_MBUS_FMT_VDIV; + struct v4l2_rect sink_y, sink_c, src_y, src_c; + struct v4l2_mbus_framefmt *src_fmt; + struct v4l2_rect *sink_crop; + + sink_crop = rkisp1_rsz_get_pad_crop(rsz, NULL, RKISP1_RSZ_PAD_SINK, + V4L2_SUBDEV_FORMAT_ACTIVE); + src_fmt = rkisp1_rsz_get_pad_fmt(rsz, NULL, RKISP1_RSZ_PAD_SRC, + V4L2_SUBDEV_FORMAT_ACTIVE); + + if (rsz->fmt_type == RKISP1_FMT_BAYER) { + rkisp1_rsz_disable(rsz, when); + return; + } + + sink_y.width = sink_crop->width; + sink_y.height = sink_crop->height; + src_y.width = src_fmt->width; + src_y.height = src_fmt->height; + + sink_c.width = sink_y.width / RKISP1_MBUS_FMT_HDIV; + sink_c.height = sink_y.height / RKISP1_MBUS_FMT_VDIV; + + if (rsz->fmt_type == RKISP1_FMT_YUV) { + struct rkisp1_capture *cap = + &rsz->rkisp1->capture_devs[rsz->id]; + const struct v4l2_format_info *pixfmt_info = + v4l2_format_info(cap->pix.fmt.pixelformat); + + hdiv = pixfmt_info->hdiv; + vdiv = pixfmt_info->vdiv; + } + src_c.width = src_y.width / hdiv; + src_c.height = src_y.height / vdiv; + + if (sink_c.width == src_c.width && sink_c.height == src_c.height) { + rkisp1_rsz_disable(rsz, when); + return; + } + + dev_dbg(rsz->rkisp1->dev, "stream %d rsz/scale: %dx%d -> %dx%d\n", + rsz->id, sink_crop->width, sink_crop->height, + src_fmt->width, src_fmt->height); + dev_dbg(rsz->rkisp1->dev, "chroma scaling %dx%d -> %dx%d\n", + sink_c.width, sink_c.height, src_c.width, src_c.height); + + /* set values in the hw */ + rkisp1_rsz_config_regs(rsz, &sink_y, &sink_c, &src_y, &src_c, when); + + rkisp1_rsz_dump_regs(rsz); +} + +/* ---------------------------------------------------------------------------- + * Subdev pad operations + */ + +static int rkisp1_rsz_enum_mbus_code(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_mbus_code_enum *code) +{ + struct rkisp1_resizer *rsz = + container_of(sd, struct rkisp1_resizer, sd); + struct v4l2_subdev_pad_config dummy_cfg; + u32 pad = code->pad; + int ret; + + /* supported mbus codes are the same in isp sink pad */ + code->pad = RKISP1_ISP_PAD_SINK_VIDEO; + ret = v4l2_subdev_call(&rsz->rkisp1->isp.sd, pad, enum_mbus_code, + &dummy_cfg, code); + + /* restore pad */ + code->pad = pad; + return ret; +} + +static int rkisp1_rsz_init_config(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg) +{ + struct v4l2_mbus_framefmt *sink_fmt, *src_fmt; + struct v4l2_rect *sink_crop; + + sink_fmt = v4l2_subdev_get_try_format(sd, cfg, RKISP1_RSZ_PAD_SRC); + sink_fmt->width = RKISP1_DEFAULT_WIDTH; + sink_fmt->height = RKISP1_DEFAULT_HEIGHT; + sink_fmt->field = V4L2_FIELD_NONE; + sink_fmt->code = RKISP1_DEF_FMT; + + sink_crop = v4l2_subdev_get_try_crop(sd, cfg, RKISP1_RSZ_PAD_SINK); + sink_crop->width = RKISP1_DEFAULT_WIDTH; + sink_crop->height = RKISP1_DEFAULT_HEIGHT; + sink_crop->left = 0; + sink_crop->top = 0; + + src_fmt = v4l2_subdev_get_try_format(sd, cfg, RKISP1_RSZ_PAD_SINK); + *src_fmt = *sink_fmt; + + /* NOTE: there is no crop in the source pad, only in the sink */ + + return 0; +} + +static void rkisp1_rsz_set_src_fmt(struct rkisp1_resizer *rsz, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_mbus_framefmt *format, + unsigned int which) +{ + struct v4l2_mbus_framefmt *src_fmt; + + src_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SRC, which); + src_fmt->width = clamp_t(u32, format->width, + rsz->config->min_rsz_width, + rsz->config->max_rsz_width); + src_fmt->height = clamp_t(u32, format->height, + rsz->config->min_rsz_height, + rsz->config->max_rsz_height); + + *format = *src_fmt; +} + +static void rkisp1_rsz_set_sink_crop(struct rkisp1_resizer *rsz, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_rect *r, + unsigned int which) +{ + const struct rkisp1_isp_mbus_info *mbus_info; + struct v4l2_mbus_framefmt *sink_fmt; + struct v4l2_rect *sink_crop; + + sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, which); + sink_crop = rkisp1_rsz_get_pad_crop(rsz, cfg, RKISP1_RSZ_PAD_SINK, + which); + + /* Not crop for MP bayer raw data */ + mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); + + if (rsz->id == RKISP1_MAINPATH && + mbus_info->fmt_type == RKISP1_FMT_BAYER) { + sink_crop->left = 0; + sink_crop->top = 0; + sink_crop->width = sink_fmt->width; + sink_crop->height = sink_fmt->height; + return; + } + + sink_crop->left = ALIGN(r->left, 2); + sink_crop->width = ALIGN(r->width, 2); + sink_crop->top = r->top; + sink_crop->height = r->height; + rkisp1_sd_adjust_crop(sink_crop, sink_fmt); + + *r = *sink_crop; +} + +static void rkisp1_rsz_set_sink_fmt(struct rkisp1_resizer *rsz, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_mbus_framefmt *format, + unsigned int which) +{ + const struct rkisp1_isp_mbus_info *mbus_info; + struct v4l2_mbus_framefmt *sink_fmt, *src_fmt; + struct v4l2_rect *sink_crop; + + sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, which); + src_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SRC, which); + sink_crop = rkisp1_rsz_get_pad_crop(rsz, cfg, RKISP1_RSZ_PAD_SINK, + which); + sink_fmt->code = format->code; + mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); + if (!mbus_info) { + sink_fmt->code = RKISP1_DEF_FMT; + mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); + } + if (which == V4L2_SUBDEV_FORMAT_ACTIVE) + rsz->fmt_type = mbus_info->fmt_type; + + if (rsz->id == RKISP1_MAINPATH && + mbus_info->fmt_type == RKISP1_FMT_BAYER) { + sink_crop->left = 0; + sink_crop->top = 0; + sink_crop->width = sink_fmt->width; + sink_crop->height = sink_fmt->height; + return; + } + + /* Propagete to source pad */ + src_fmt->code = sink_fmt->code; + + sink_fmt->width = clamp_t(u32, format->width, + rsz->config->min_rsz_width, + rsz->config->max_rsz_width); + sink_fmt->height = clamp_t(u32, format->height, + rsz->config->min_rsz_height, + rsz->config->max_rsz_height); + + *format = *sink_fmt; + + /* Update sink crop */ + rkisp1_rsz_set_sink_crop(rsz, cfg, sink_crop, which); +} + +static int rkisp1_rsz_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_format *fmt) +{ + struct rkisp1_resizer *rsz = + container_of(sd, struct rkisp1_resizer, sd); + + fmt->format = *rkisp1_rsz_get_pad_fmt(rsz, cfg, fmt->pad, fmt->which); + return 0; +} + +static int rkisp1_rsz_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_format *fmt) +{ + struct rkisp1_resizer *rsz = + container_of(sd, struct rkisp1_resizer, sd); + + if (fmt->pad == RKISP1_RSZ_PAD_SINK) + rkisp1_rsz_set_sink_fmt(rsz, cfg, &fmt->format, fmt->which); + else + rkisp1_rsz_set_src_fmt(rsz, cfg, &fmt->format, fmt->which); + + return 0; +} + +static int rkisp1_rsz_get_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_selection *sel) +{ + struct rkisp1_resizer *rsz = + container_of(sd, struct rkisp1_resizer, sd); + struct v4l2_mbus_framefmt *mf_sink; + + if (sel->pad == RKISP1_RSZ_PAD_SRC) + return -EINVAL; + + switch (sel->target) { + case V4L2_SEL_TGT_CROP_BOUNDS: + mf_sink = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, + sel->which); + sel->r.height = mf_sink->height; + sel->r.width = mf_sink->width; + sel->r.left = 0; + sel->r.top = 0; + break; + case V4L2_SEL_TGT_CROP: + sel->r = *rkisp1_rsz_get_pad_crop(rsz, cfg, RKISP1_RSZ_PAD_SINK, + sel->which); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int rkisp1_rsz_set_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_selection *sel) +{ + struct rkisp1_resizer *rsz = + container_of(sd, struct rkisp1_resizer, sd); + + if (sel->target != V4L2_SEL_TGT_CROP || sel->pad == RKISP1_RSZ_PAD_SRC) + return -EINVAL; + + dev_dbg(sd->dev, "%s: pad: %d sel(%d,%d)/%dx%d\n", __func__, + sel->pad, sel->r.left, sel->r.top, sel->r.width, sel->r.height); + + rkisp1_rsz_set_sink_crop(rsz, cfg, &sel->r, sel->which); + + return 0; +} + +static const struct media_entity_operations rkisp1_rsz_media_ops = { + .link_validate = v4l2_subdev_link_validate, +}; + +static const struct v4l2_subdev_pad_ops rkisp1_rsz_pad_ops = { + .enum_mbus_code = rkisp1_rsz_enum_mbus_code, + .get_selection = rkisp1_rsz_get_selection, + .set_selection = rkisp1_rsz_set_selection, + .init_cfg = rkisp1_rsz_init_config, + .get_fmt = rkisp1_rsz_get_fmt, + .set_fmt = rkisp1_rsz_set_fmt, + .link_validate = v4l2_subdev_link_validate_default, +}; + +/* ---------------------------------------------------------------------------- + * Stream operations + */ + +static int rkisp1_rsz_s_stream(struct v4l2_subdev *sd, int enable) +{ + struct rkisp1_resizer *rsz = + container_of(sd, struct rkisp1_resizer, sd); + struct rkisp1_device *rkisp1 = rsz->rkisp1; + struct rkisp1_capture *other = &rkisp1->capture_devs[rsz->id ^ 1]; + enum rkisp1_shadow_regs_when when = RKISP1_SHADOW_REGS_SYNC; + + if (!enable) { + rkisp1_dcrop_disable(rsz, RKISP1_SHADOW_REGS_ASYNC); + rkisp1_rsz_disable(rsz, RKISP1_SHADOW_REGS_ASYNC); + return 0; + } + + if (other->is_streaming) + when = RKISP1_SHADOW_REGS_ASYNC; + + rkisp1_rsz_config(rsz, when); + rkisp1_dcrop_config(rsz); + + return 0; +} + +static const struct v4l2_subdev_video_ops rkisp1_rsz_video_ops = { + .s_stream = rkisp1_rsz_s_stream, +}; + +static const struct v4l2_subdev_ops rkisp1_rsz_ops = { + .video = &rkisp1_rsz_video_ops, + .pad = &rkisp1_rsz_pad_ops, +}; + +static void rkisp1_rsz_unregister(struct rkisp1_resizer *rsz) +{ + v4l2_device_unregister_subdev(&rsz->sd); + media_entity_cleanup(&rsz->sd.entity); +} + +static int rkisp1_rsz_register(struct rkisp1_resizer *rsz) +{ + const char * const dev_names[] = {RKISP1_RSZ_MP_DEV_NAME, + RKISP1_RSZ_SP_DEV_NAME}; + struct media_pad *pads = rsz->pads; + struct v4l2_subdev *sd = &rsz->sd; + int ret; + + if (rsz->id == RKISP1_SELFPATH) + rsz->config = &rkisp1_rsz_config_sp; + else + rsz->config = &rkisp1_rsz_config_mp; + + v4l2_subdev_init(sd, &rkisp1_rsz_ops); + sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; + sd->entity.ops = &rkisp1_rsz_media_ops; + sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_SCALER; + sd->owner = THIS_MODULE; + strscpy(sd->name, dev_names[rsz->id], sizeof(sd->name)); + + pads[RKISP1_RSZ_PAD_SINK].flags = MEDIA_PAD_FL_SINK | + MEDIA_PAD_FL_MUST_CONNECT; + pads[RKISP1_RSZ_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE | + MEDIA_PAD_FL_MUST_CONNECT; + + rsz->fmt_type = RKISP1_DEF_FMT_TYPE; + + ret = media_entity_pads_init(&sd->entity, 2, pads); + if (ret) + return ret; + + ret = v4l2_device_register_subdev(&rsz->rkisp1->v4l2_dev, sd); + if (ret) { + dev_err(sd->dev, "Failed to register resizer subdev\n"); + goto err_cleanup_media_entity; + } + + rkisp1_rsz_init_config(sd, rsz->pad_cfg); + return 0; + +err_cleanup_media_entity: + media_entity_cleanup(&sd->entity); + + return ret; +} + +int rkisp1_resizer_devs_register(struct rkisp1_device *rkisp1) +{ + struct rkisp1_resizer *rsz; + unsigned int i, j; + int ret; + + for (i = 0; i < ARRAY_SIZE(rkisp1->resizer_devs); i++) { + rsz = &rkisp1->resizer_devs[i]; + rsz->rkisp1 = rkisp1; + rsz->id = i; + ret = rkisp1_rsz_register(rsz); + if (ret) + goto err_unreg_resizer_devs; + } + + return 0; + +err_unreg_resizer_devs: + for (j = 0; j < i; j++) { + rsz = &rkisp1->resizer_devs[j]; + rkisp1_rsz_unregister(rsz); + } + + return ret; +} + +void rkisp1_resizer_devs_unregister(struct rkisp1_device *rkisp1) +{ + struct rkisp1_resizer *mp = &rkisp1->resizer_devs[RKISP1_MAINPATH]; + struct rkisp1_resizer *sp = &rkisp1->resizer_devs[RKISP1_SELFPATH]; + + rkisp1_rsz_unregister(mp); + rkisp1_rsz_unregister(sp); +} diff --git a/drivers/staging/media/rkisp1/rkisp1-stats.c b/drivers/staging/media/rkisp1/rkisp1-stats.c new file mode 100644 index 000000000000..d98ea15837de --- /dev/null +++ b/drivers/staging/media/rkisp1/rkisp1-stats.c @@ -0,0 +1,530 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Rockchip ISP1 Driver - Stats subdevice + * + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +#include <media/v4l2-common.h> +#include <media/v4l2-event.h> +#include <media/v4l2-ioctl.h> +#include <media/videobuf2-core.h> +#include <media/videobuf2-vmalloc.h> /* for ISP statistics */ + +#include "rkisp1-common.h" + +#define RKISP1_STATS_DEV_NAME RKISP1_DRIVER_NAME "_stats" + +#define RKISP1_ISP_STATS_REQ_BUFS_MIN 2 +#define RKISP1_ISP_STATS_REQ_BUFS_MAX 8 + +enum rkisp1_isp_readout_cmd { + RKISP1_ISP_READOUT_MEAS, + RKISP1_ISP_READOUT_META, +}; + +struct rkisp1_isp_readout_work { + struct work_struct work; + struct rkisp1_stats *stats; + + unsigned int frame_id; + unsigned int isp_ris; + enum rkisp1_isp_readout_cmd readout; + struct vb2_buffer *vb; +}; + +static int rkisp1_stats_enum_fmt_meta_cap(struct file *file, void *priv, + struct v4l2_fmtdesc *f) +{ + struct video_device *video = video_devdata(file); + struct rkisp1_stats *stats = video_get_drvdata(video); + + if (f->index > 0 || f->type != video->queue->type) + return -EINVAL; + + f->pixelformat = stats->vdev_fmt.fmt.meta.dataformat; + return 0; +} + +static int rkisp1_stats_g_fmt_meta_cap(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct video_device *video = video_devdata(file); + struct rkisp1_stats *stats = video_get_drvdata(video); + struct v4l2_meta_format *meta = &f->fmt.meta; + + if (f->type != video->queue->type) + return -EINVAL; + + memset(meta, 0, sizeof(*meta)); + meta->dataformat = stats->vdev_fmt.fmt.meta.dataformat; + meta->buffersize = stats->vdev_fmt.fmt.meta.buffersize; + + return 0; +} + +static int rkisp1_stats_querycap(struct file *file, + void *priv, struct v4l2_capability *cap) +{ + struct video_device *vdev = video_devdata(file); + + strscpy(cap->driver, RKISP1_DRIVER_NAME, sizeof(cap->driver)); + strscpy(cap->card, vdev->name, sizeof(cap->card)); + strscpy(cap->bus_info, "platform: " RKISP1_DRIVER_NAME, + sizeof(cap->bus_info)); + + return 0; +} + +/* ISP video device IOCTLs */ +static const struct v4l2_ioctl_ops rkisp1_stats_ioctl = { + .vidioc_reqbufs = vb2_ioctl_reqbufs, + .vidioc_querybuf = vb2_ioctl_querybuf, + .vidioc_create_bufs = vb2_ioctl_create_bufs, + .vidioc_qbuf = vb2_ioctl_qbuf, + .vidioc_dqbuf = vb2_ioctl_dqbuf, + .vidioc_prepare_buf = vb2_ioctl_prepare_buf, + .vidioc_expbuf = vb2_ioctl_expbuf, + .vidioc_streamon = vb2_ioctl_streamon, + .vidioc_streamoff = vb2_ioctl_streamoff, + .vidioc_enum_fmt_meta_cap = rkisp1_stats_enum_fmt_meta_cap, + .vidioc_g_fmt_meta_cap = rkisp1_stats_g_fmt_meta_cap, + .vidioc_s_fmt_meta_cap = rkisp1_stats_g_fmt_meta_cap, + .vidioc_try_fmt_meta_cap = rkisp1_stats_g_fmt_meta_cap, + .vidioc_querycap = rkisp1_stats_querycap, + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +static const struct v4l2_file_operations rkisp1_stats_fops = { + .mmap = vb2_fop_mmap, + .unlocked_ioctl = video_ioctl2, + .poll = vb2_fop_poll, + .open = v4l2_fh_open, + .release = vb2_fop_release +}; + +static int rkisp1_stats_vb2_queue_setup(struct vb2_queue *vq, + unsigned int *num_buffers, + unsigned int *num_planes, + unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct rkisp1_stats *stats = vq->drv_priv; + + *num_planes = 1; + + *num_buffers = clamp_t(u32, *num_buffers, RKISP1_ISP_STATS_REQ_BUFS_MIN, + RKISP1_ISP_STATS_REQ_BUFS_MAX); + + sizes[0] = sizeof(struct rkisp1_stat_buffer); + + INIT_LIST_HEAD(&stats->stat); + + return 0; +} + +static void rkisp1_stats_vb2_buf_queue(struct vb2_buffer *vb) +{ + struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); + struct rkisp1_buffer *stats_buf = + container_of(vbuf, struct rkisp1_buffer, vb); + struct vb2_queue *vq = vb->vb2_queue; + struct rkisp1_stats *stats_dev = vq->drv_priv; + + stats_buf->vaddr[0] = vb2_plane_vaddr(vb, 0); + + mutex_lock(&stats_dev->wq_lock); + list_add_tail(&stats_buf->queue, &stats_dev->stat); + mutex_unlock(&stats_dev->wq_lock); +} + +static int rkisp1_stats_vb2_buf_prepare(struct vb2_buffer *vb) +{ + if (vb2_plane_size(vb, 0) < sizeof(struct rkisp1_stat_buffer)) + return -EINVAL; + + vb2_set_plane_payload(vb, 0, sizeof(struct rkisp1_stat_buffer)); + + return 0; +} + +static void rkisp1_stats_vb2_stop_streaming(struct vb2_queue *vq) +{ + struct rkisp1_stats *stats = vq->drv_priv; + struct rkisp1_buffer *buf; + unsigned long flags; + unsigned int i; + + /* Make sure no new work queued in isr before draining wq */ + spin_lock_irqsave(&stats->irq_lock, flags); + stats->is_streaming = false; + spin_unlock_irqrestore(&stats->irq_lock, flags); + + drain_workqueue(stats->readout_wq); + + mutex_lock(&stats->wq_lock); + for (i = 0; i < RKISP1_ISP_STATS_REQ_BUFS_MAX; i++) { + if (list_empty(&stats->stat)) + break; + buf = list_first_entry(&stats->stat, + struct rkisp1_buffer, queue); + list_del(&buf->queue); + vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR); + } + mutex_unlock(&stats->wq_lock); +} + +static int +rkisp1_stats_vb2_start_streaming(struct vb2_queue *queue, unsigned int count) +{ + struct rkisp1_stats *stats = queue->drv_priv; + + stats->is_streaming = true; + + return 0; +} + +static const struct vb2_ops rkisp1_stats_vb2_ops = { + .queue_setup = rkisp1_stats_vb2_queue_setup, + .buf_queue = rkisp1_stats_vb2_buf_queue, + .buf_prepare = rkisp1_stats_vb2_buf_prepare, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, + .stop_streaming = rkisp1_stats_vb2_stop_streaming, + .start_streaming = rkisp1_stats_vb2_start_streaming, +}; + +static int +rkisp1_stats_init_vb2_queue(struct vb2_queue *q, struct rkisp1_stats *stats) +{ + struct rkisp1_vdev_node *node; + + node = container_of(q, struct rkisp1_vdev_node, buf_queue); + + q->type = V4L2_BUF_TYPE_META_CAPTURE; + q->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF; + q->drv_priv = stats; + q->ops = &rkisp1_stats_vb2_ops; + q->mem_ops = &vb2_vmalloc_memops; + q->buf_struct_size = sizeof(struct rkisp1_buffer); + q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + q->lock = &node->vlock; + + return vb2_queue_init(q); +} + +static void rkisp1_stats_get_awb_meas(struct rkisp1_stats *stats, + struct rkisp1_stat_buffer *pbuf) +{ + /* Protect against concurrent access from ISR? */ + struct rkisp1_device *rkisp1 = stats->rkisp1; + u32 reg_val; + + pbuf->meas_type |= RKISP1_CIF_ISP_STAT_AWB; + reg_val = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AWB_WHITE_CNT); + pbuf->params.awb.awb_mean[0].cnt = + RKISP1_CIF_ISP_AWB_GET_PIXEL_CNT(reg_val); + reg_val = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AWB_MEAN); + + pbuf->params.awb.awb_mean[0].mean_cr_or_r = + RKISP1_CIF_ISP_AWB_GET_MEAN_CR_R(reg_val); + pbuf->params.awb.awb_mean[0].mean_cb_or_b = + RKISP1_CIF_ISP_AWB_GET_MEAN_CB_B(reg_val); + pbuf->params.awb.awb_mean[0].mean_y_or_g = + RKISP1_CIF_ISP_AWB_GET_MEAN_Y_G(reg_val); +} + +static void rkisp1_stats_get_aec_meas(struct rkisp1_stats *stats, + struct rkisp1_stat_buffer *pbuf) +{ + struct rkisp1_device *rkisp1 = stats->rkisp1; + unsigned int i; + + pbuf->meas_type |= RKISP1_CIF_ISP_STAT_AUTOEXP; + for (i = 0; i < RKISP1_CIF_ISP_AE_MEAN_MAX; i++) + pbuf->params.ae.exp_mean[i] = + (u8)rkisp1_read(rkisp1, + RKISP1_CIF_ISP_EXP_MEAN_00 + i * 4); +} + +static void rkisp1_stats_get_afc_meas(struct rkisp1_stats *stats, + struct rkisp1_stat_buffer *pbuf) +{ + struct rkisp1_device *rkisp1 = stats->rkisp1; + struct rkisp1_cif_isp_af_stat *af; + + pbuf->meas_type = RKISP1_CIF_ISP_STAT_AFM_FIN; + + af = &pbuf->params.af; + af->window[0].sum = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AFM_SUM_A); + af->window[0].lum = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AFM_LUM_A); + af->window[1].sum = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AFM_SUM_B); + af->window[1].lum = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AFM_LUM_B); + af->window[2].sum = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AFM_SUM_C); + af->window[2].lum = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AFM_LUM_C); +} + +static void rkisp1_stats_get_hst_meas(struct rkisp1_stats *stats, + struct rkisp1_stat_buffer *pbuf) +{ + struct rkisp1_device *rkisp1 = stats->rkisp1; + unsigned int i; + + pbuf->meas_type |= RKISP1_CIF_ISP_STAT_HIST; + for (i = 0; i < RKISP1_CIF_ISP_HIST_BIN_N_MAX; i++) + pbuf->params.hist.hist_bins[i] = + (u8)rkisp1_read(rkisp1, + RKISP1_CIF_ISP_HIST_BIN_0 + i * 4); +} + +static void rkisp1_stats_get_bls_meas(struct rkisp1_stats *stats, + struct rkisp1_stat_buffer *pbuf) +{ + struct rkisp1_device *rkisp1 = stats->rkisp1; + const struct rkisp1_isp_mbus_info *in_fmt = rkisp1->isp.sink_fmt; + struct rkisp1_cif_isp_bls_meas_val *bls_val; + + bls_val = &pbuf->params.ae.bls_val; + if (in_fmt->bayer_pat == RKISP1_RAW_BGGR) { + bls_val->meas_b = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_A_MEASURED); + bls_val->meas_gb = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_B_MEASURED); + bls_val->meas_gr = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_C_MEASURED); + bls_val->meas_r = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_D_MEASURED); + } else if (in_fmt->bayer_pat == RKISP1_RAW_GBRG) { + bls_val->meas_gb = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_A_MEASURED); + bls_val->meas_b = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_B_MEASURED); + bls_val->meas_r = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_C_MEASURED); + bls_val->meas_gr = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_D_MEASURED); + } else if (in_fmt->bayer_pat == RKISP1_RAW_GRBG) { + bls_val->meas_gr = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_A_MEASURED); + bls_val->meas_r = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_B_MEASURED); + bls_val->meas_b = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_C_MEASURED); + bls_val->meas_gb = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_D_MEASURED); + } else if (in_fmt->bayer_pat == RKISP1_RAW_RGGB) { + bls_val->meas_r = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_A_MEASURED); + bls_val->meas_gr = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_B_MEASURED); + bls_val->meas_gb = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_C_MEASURED); + bls_val->meas_b = + rkisp1_read(rkisp1, RKISP1_CIF_ISP_BLS_D_MEASURED); + } +} + +static void +rkisp1_stats_send_measurement(struct rkisp1_stats *stats, + struct rkisp1_isp_readout_work *meas_work) +{ + struct rkisp1_stat_buffer *cur_stat_buf; + struct rkisp1_buffer *cur_buf = NULL; + unsigned int frame_sequence = + atomic_read(&stats->rkisp1->isp.frame_sequence); + u64 timestamp = ktime_get_ns(); + + if (frame_sequence != meas_work->frame_id) { + dev_warn(stats->rkisp1->dev, + "Measurement late(%d, %d)\n", + frame_sequence, meas_work->frame_id); + frame_sequence = meas_work->frame_id; + } + + mutex_lock(&stats->wq_lock); + /* get one empty buffer */ + if (!list_empty(&stats->stat)) { + cur_buf = list_first_entry(&stats->stat, + struct rkisp1_buffer, queue); + list_del(&cur_buf->queue); + } + mutex_unlock(&stats->wq_lock); + + if (!cur_buf) + return; + + cur_stat_buf = + (struct rkisp1_stat_buffer *)(cur_buf->vaddr[0]); + + if (meas_work->isp_ris & RKISP1_CIF_ISP_AWB_DONE) { + rkisp1_stats_get_awb_meas(stats, cur_stat_buf); + cur_stat_buf->meas_type |= RKISP1_CIF_ISP_STAT_AWB; + } + + if (meas_work->isp_ris & RKISP1_CIF_ISP_AFM_FIN) { + rkisp1_stats_get_afc_meas(stats, cur_stat_buf); + cur_stat_buf->meas_type |= RKISP1_CIF_ISP_STAT_AFM_FIN; + } + + if (meas_work->isp_ris & RKISP1_CIF_ISP_EXP_END) { + rkisp1_stats_get_aec_meas(stats, cur_stat_buf); + rkisp1_stats_get_bls_meas(stats, cur_stat_buf); + cur_stat_buf->meas_type |= RKISP1_CIF_ISP_STAT_AUTOEXP; + } + + if (meas_work->isp_ris & RKISP1_CIF_ISP_HIST_MEASURE_RDY) { + rkisp1_stats_get_hst_meas(stats, cur_stat_buf); + cur_stat_buf->meas_type |= RKISP1_CIF_ISP_STAT_HIST; + } + + vb2_set_plane_payload(&cur_buf->vb.vb2_buf, 0, + sizeof(struct rkisp1_stat_buffer)); + cur_buf->vb.sequence = frame_sequence; + cur_buf->vb.vb2_buf.timestamp = timestamp; + vb2_buffer_done(&cur_buf->vb.vb2_buf, VB2_BUF_STATE_DONE); +} + +static void rkisp1_stats_readout_work(struct work_struct *work) +{ + struct rkisp1_isp_readout_work *readout_work = + container_of(work, struct rkisp1_isp_readout_work, work); + struct rkisp1_stats *stats = readout_work->stats; + + if (readout_work->readout == RKISP1_ISP_READOUT_MEAS) + rkisp1_stats_send_measurement(stats, readout_work); + + kfree(readout_work); +} + +void rkisp1_stats_isr(struct rkisp1_stats *stats, u32 isp_ris) +{ + unsigned int frame_sequence = + atomic_read(&stats->rkisp1->isp.frame_sequence); + struct rkisp1_device *rkisp1 = stats->rkisp1; + struct rkisp1_isp_readout_work *work; + unsigned int isp_mis_tmp = 0; + u32 val; + + spin_lock(&stats->irq_lock); + + val = RKISP1_CIF_ISP_AWB_DONE | RKISP1_CIF_ISP_AFM_FIN | + RKISP1_CIF_ISP_EXP_END | RKISP1_CIF_ISP_HIST_MEASURE_RDY; + rkisp1_write(rkisp1, val, RKISP1_CIF_ISP_ICR); + + isp_mis_tmp = rkisp1_read(rkisp1, RKISP1_CIF_ISP_MIS); + if (isp_mis_tmp & + (RKISP1_CIF_ISP_AWB_DONE | RKISP1_CIF_ISP_AFM_FIN | + RKISP1_CIF_ISP_EXP_END | RKISP1_CIF_ISP_HIST_MEASURE_RDY)) + rkisp1->debug.stats_error++; + + if (!stats->is_streaming) + goto unlock; + if (isp_ris & (RKISP1_CIF_ISP_AWB_DONE | + RKISP1_CIF_ISP_AFM_FIN | + RKISP1_CIF_ISP_EXP_END | + RKISP1_CIF_ISP_HIST_MEASURE_RDY)) { + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (work) { + INIT_WORK(&work->work, + rkisp1_stats_readout_work); + work->readout = RKISP1_ISP_READOUT_MEAS; + work->stats = stats; + work->frame_id = frame_sequence; + work->isp_ris = isp_ris; + if (!queue_work(stats->readout_wq, + &work->work)) + kfree(work); + } else { + dev_err(stats->rkisp1->dev, + "Could not allocate work\n"); + } + } + +unlock: + spin_unlock(&stats->irq_lock); +} + +static void rkisp1_init_stats(struct rkisp1_stats *stats) +{ + stats->vdev_fmt.fmt.meta.dataformat = + V4L2_META_FMT_RK_ISP1_STAT_3A; + stats->vdev_fmt.fmt.meta.buffersize = + sizeof(struct rkisp1_stat_buffer); +} + +int rkisp1_stats_register(struct rkisp1_stats *stats, + struct v4l2_device *v4l2_dev, + struct rkisp1_device *rkisp1) +{ + struct rkisp1_vdev_node *node = &stats->vnode; + struct video_device *vdev = &node->vdev; + int ret; + + stats->rkisp1 = rkisp1; + mutex_init(&stats->wq_lock); + mutex_init(&node->vlock); + INIT_LIST_HEAD(&stats->stat); + spin_lock_init(&stats->irq_lock); + + strscpy(vdev->name, RKISP1_STATS_DEV_NAME, sizeof(vdev->name)); + + video_set_drvdata(vdev, stats); + vdev->ioctl_ops = &rkisp1_stats_ioctl; + vdev->fops = &rkisp1_stats_fops; + vdev->release = video_device_release_empty; + vdev->lock = &node->vlock; + vdev->v4l2_dev = v4l2_dev; + vdev->queue = &node->buf_queue; + vdev->device_caps = V4L2_CAP_META_CAPTURE | V4L2_CAP_STREAMING; + vdev->vfl_dir = VFL_DIR_RX; + rkisp1_stats_init_vb2_queue(vdev->queue, stats); + rkisp1_init_stats(stats); + video_set_drvdata(vdev, stats); + + node->pad.flags = MEDIA_PAD_FL_SINK; + ret = media_entity_pads_init(&vdev->entity, 1, &node->pad); + if (ret) + goto err_release_queue; + + ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1); + if (ret) { + dev_err(&vdev->dev, + "failed to register %s, ret=%d\n", vdev->name, ret); + goto err_cleanup_media_entity; + } + + stats->readout_wq = alloc_workqueue("measurement_queue", + WQ_UNBOUND | WQ_MEM_RECLAIM, + 1); + + if (!stats->readout_wq) { + ret = -ENOMEM; + goto err_unreg_vdev; + } + + return 0; + +err_unreg_vdev: + video_unregister_device(vdev); +err_cleanup_media_entity: + media_entity_cleanup(&vdev->entity); +err_release_queue: + vb2_queue_release(vdev->queue); + mutex_destroy(&node->vlock); + mutex_destroy(&stats->wq_lock); + return ret; +} + +void rkisp1_stats_unregister(struct rkisp1_stats *stats) +{ + struct rkisp1_vdev_node *node = &stats->vnode; + struct video_device *vdev = &node->vdev; + + destroy_workqueue(stats->readout_wq); + video_unregister_device(vdev); + media_entity_cleanup(&vdev->entity); + vb2_queue_release(vdev->queue); + mutex_destroy(&node->vlock); + mutex_destroy(&stats->wq_lock); +} diff --git a/drivers/staging/media/rkisp1/uapi/rkisp1-config.h b/drivers/staging/media/rkisp1/uapi/rkisp1-config.h new file mode 100644 index 000000000000..ca0d031b14ac --- /dev/null +++ b/drivers/staging/media/rkisp1/uapi/rkisp1-config.h @@ -0,0 +1,819 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Rockchip ISP1 userspace API + * Copyright (C) 2017 Rockchip Electronics Co., Ltd. + */ + +/* + * TODO: Improve documentation, mostly regarding abbreviation and hardware + * specificities. Reference: "REF_01 - ISP_user_manual, Rev 2.57" (not public) + */ + +#ifndef _UAPI_RKISP1_CONFIG_H +#define _UAPI_RKISP1_CONFIG_H + +#include <linux/types.h> + +/* Vendor specific - used for RK_ISP1 camera sub-system */ +#define V4L2_META_FMT_RK_ISP1_PARAMS v4l2_fourcc('R', 'K', '1', 'P') /* Rockchip ISP1 params */ +#define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S') /* Rockchip ISP1 3A statistics */ + +#define RKISP1_CIF_ISP_MODULE_DPCC BIT(0) +#define RKISP1_CIF_ISP_MODULE_BLS BIT(1) +#define RKISP1_CIF_ISP_MODULE_SDG BIT(2) +#define RKISP1_CIF_ISP_MODULE_HST BIT(3) +#define RKISP1_CIF_ISP_MODULE_LSC BIT(4) +#define RKISP1_CIF_ISP_MODULE_AWB_GAIN BIT(5) +#define RKISP1_CIF_ISP_MODULE_FLT BIT(6) +#define RKISP1_CIF_ISP_MODULE_BDM BIT(7) +#define RKISP1_CIF_ISP_MODULE_CTK BIT(8) +#define RKISP1_CIF_ISP_MODULE_GOC BIT(9) +#define RKISP1_CIF_ISP_MODULE_CPROC BIT(10) +#define RKISP1_CIF_ISP_MODULE_AFC BIT(11) +#define RKISP1_CIF_ISP_MODULE_AWB BIT(12) +#define RKISP1_CIF_ISP_MODULE_IE BIT(13) +#define RKISP1_CIF_ISP_MODULE_AEC BIT(14) +#define RKISP1_CIF_ISP_MODULE_WDR BIT(15) +#define RKISP1_CIF_ISP_MODULE_DPF BIT(16) +#define RKISP1_CIF_ISP_MODULE_DPF_STRENGTH BIT(17) + +#define RKISP1_CIF_ISP_CTK_COEFF_MAX 0x100 +#define RKISP1_CIF_ISP_CTK_OFFSET_MAX 0x800 + +#define RKISP1_CIF_ISP_AE_MEAN_MAX 25 +#define RKISP1_CIF_ISP_HIST_BIN_N_MAX 16 +#define RKISP1_CIF_ISP_AFM_MAX_WINDOWS 3 +#define RKISP1_CIF_ISP_DEGAMMA_CURVE_SIZE 17 + +#define RKISP1_CIF_ISP_BDM_MAX_TH 0xff + +/* + * Black level compensation + */ +/* maximum value for horizontal start address */ +#define RKISP1_CIF_ISP_BLS_START_H_MAX 0x00000fff +/* maximum value for horizontal stop address */ +#define RKISP1_CIF_ISP_BLS_STOP_H_MAX 0x00000fff +/* maximum value for vertical start address */ +#define RKISP1_CIF_ISP_BLS_START_V_MAX 0x00000fff +/* maximum value for vertical stop address */ +#define RKISP1_CIF_ISP_BLS_STOP_V_MAX 0x00000fff +/* maximum is 2^18 = 262144*/ +#define RKISP1_CIF_ISP_BLS_SAMPLES_MAX 0x00000012 +/* maximum value for fixed black level */ +#define RKISP1_CIF_ISP_BLS_FIX_SUB_MAX 0x00000fff +/* minimum value for fixed black level */ +#define RKISP1_CIF_ISP_BLS_FIX_SUB_MIN 0xfffff000 +/* 13 bit range (signed)*/ +#define RKISP1_CIF_ISP_BLS_FIX_MASK 0x00001fff + +/* + * Automatic white balance measurments + */ +#define RKISP1_CIF_ISP_AWB_MAX_GRID 1 +#define RKISP1_CIF_ISP_AWB_MAX_FRAMES 7 + +/* + * Gamma out + */ +/* Maximum number of color samples supported */ +#define RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES 17 + +/* + * Lens shade correction + */ +#define RKISP1_CIF_ISP_LSC_GRAD_TBL_SIZE 8 +#define RKISP1_CIF_ISP_LSC_SIZE_TBL_SIZE 8 +/* + * The following matches the tuning process, + * not the max capabilities of the chip. + * Last value unused. + */ +#define RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE 290 + +/* + * Histogram calculation + */ +/* Last 3 values unused. */ +#define RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE 28 + +/* + * Defect Pixel Cluster Correction + */ +#define RKISP1_CIF_ISP_DPCC_METHODS_MAX 3 + +/* + * Denoising pre filter + */ +#define RKISP1_CIF_ISP_DPF_MAX_NLF_COEFFS 17 +#define RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS 6 + +/* + * Measurement types + */ +#define RKISP1_CIF_ISP_STAT_AWB BIT(0) +#define RKISP1_CIF_ISP_STAT_AUTOEXP BIT(1) +#define RKISP1_CIF_ISP_STAT_AFM_FIN BIT(2) +#define RKISP1_CIF_ISP_STAT_HIST BIT(3) + +enum rkisp1_cif_isp_histogram_mode { + RKISP1_CIF_ISP_HISTOGRAM_MODE_DISABLE, + RKISP1_CIF_ISP_HISTOGRAM_MODE_RGB_COMBINED, + RKISP1_CIF_ISP_HISTOGRAM_MODE_R_HISTOGRAM, + RKISP1_CIF_ISP_HISTOGRAM_MODE_G_HISTOGRAM, + RKISP1_CIF_ISP_HISTOGRAM_MODE_B_HISTOGRAM, + RKISP1_CIF_ISP_HISTOGRAM_MODE_Y_HISTOGRAM +}; + +enum rkisp1_cif_isp_awb_mode_type { + RKISP1_CIF_ISP_AWB_MODE_MANUAL, + RKISP1_CIF_ISP_AWB_MODE_RGB, + RKISP1_CIF_ISP_AWB_MODE_YCBCR +}; + +enum rkisp1_cif_isp_flt_mode { + RKISP1_CIF_ISP_FLT_STATIC_MODE, + RKISP1_CIF_ISP_FLT_DYNAMIC_MODE +}; + +/** + * enum rkisp1_cif_isp_exp_ctrl_autostop - stop modes + * @RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0: continuous measurement + * @RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_1: stop measuring after a complete frame + */ +enum rkisp1_cif_isp_exp_ctrl_autostop { + RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0 = 0, + RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_1 = 1, +}; + +/** + * enum rkisp1_cif_isp_exp_meas_mode - Exposure measure mode + * @RKISP1_CIF_ISP_EXP_MEASURING_MODE_0: Y = 16 + 0.25R + 0.5G + 0.1094B + * @RKISP1_CIF_ISP_EXP_MEASURING_MODE_1: Y = (R + G + B) x (85/256) + */ +enum rkisp1_cif_isp_exp_meas_mode { + RKISP1_CIF_ISP_EXP_MEASURING_MODE_0, + RKISP1_CIF_ISP_EXP_MEASURING_MODE_1, +}; + +/*---------- PART1: Input Parameters ------------*/ + +struct rkisp1_cif_isp_window { + __u16 h_offs; + __u16 v_offs; + __u16 h_size; + __u16 v_size; +} __packed; + +/** + * struct rkisp1_cif_isp_bls_fixed_val - BLS fixed subtraction values + * + * The values will be subtracted from the sensor + * values. Therefore a negative value means addition instead of subtraction! + * + * @r: Fixed (signed!) subtraction value for Bayer pattern R + * @gr: Fixed (signed!) subtraction value for Bayer pattern Gr + * @gb: Fixed (signed!) subtraction value for Bayer pattern Gb + * @b: Fixed (signed!) subtraction value for Bayer pattern B + */ +struct rkisp1_cif_isp_bls_fixed_val { + __s16 r; + __s16 gr; + __s16 gb; + __s16 b; +} __packed; + +/** + * struct rkisp1_cif_isp_bls_config - Configuration used by black level subtraction + * + * @enable_auto: Automatic mode activated means that the measured values + * are subtracted. Otherwise the fixed subtraction + * values will be subtracted. + * @en_windows: enabled window + * @bls_window1: Measurement window 1 size + * @bls_window2: Measurement window 2 size + * @bls_samples: Set amount of measured pixels for each Bayer position + * (A, B,C and D) to 2^bls_samples. + * @fixed_val: Fixed subtraction values + */ +struct rkisp1_cif_isp_bls_config { + __u8 enable_auto; + __u8 en_windows; + struct rkisp1_cif_isp_window bls_window1; + struct rkisp1_cif_isp_window bls_window2; + __u8 bls_samples; + struct rkisp1_cif_isp_bls_fixed_val fixed_val; +} __packed; + +/** + * struct rkisp1_cif_isp_dpcc_methods_config - Methods Configuration used by DPCC + * + * Methods Configuration used by Defect Pixel Cluster Correction + * + * @method: Method enable bits + * @line_thresh: Line threshold + * @line_mad_fac: Line MAD factor + * @pg_fac: Peak gradient factor + * @rnd_thresh: Rank Neighbor Difference threshold + * @rg_fac: Rank gradient factor + */ +struct rkisp1_cif_isp_dpcc_methods_config { + __u32 method; + __u32 line_thresh; + __u32 line_mad_fac; + __u32 pg_fac; + __u32 rnd_thresh; + __u32 rg_fac; +} __packed; + +/** + * struct rkisp1_cif_isp_dpcc_config - Configuration used by DPCC + * + * Configuration used by Defect Pixel Cluster Correction + * + * @mode: dpcc output mode + * @output_mode: whether use hard coded methods + * @set_use: stage1 methods set + * @methods: methods config + * @ro_limits: rank order limits + * @rnd_offs: differential rank offsets for rank neighbor difference + */ +struct rkisp1_cif_isp_dpcc_config { + __u32 mode; + __u32 output_mode; + __u32 set_use; + struct rkisp1_cif_isp_dpcc_methods_config methods[RKISP1_CIF_ISP_DPCC_METHODS_MAX]; + __u32 ro_limits; + __u32 rnd_offs; +} __packed; + +struct rkisp1_cif_isp_gamma_corr_curve { + __u16 gamma_y[RKISP1_CIF_ISP_DEGAMMA_CURVE_SIZE]; +} __packed; + +struct rkisp1_cif_isp_gamma_curve_x_axis_pnts { + __u32 gamma_dx0; + __u32 gamma_dx1; +} __packed; + +/** + * struct rkisp1_cif_isp_sdg_config - Configuration used by sensor degamma + * + * @curve_x: gamma curve point definition axis for x + * @xa_pnts: x increments + */ +struct rkisp1_cif_isp_sdg_config { + struct rkisp1_cif_isp_gamma_corr_curve curve_r; + struct rkisp1_cif_isp_gamma_corr_curve curve_g; + struct rkisp1_cif_isp_gamma_corr_curve curve_b; + struct rkisp1_cif_isp_gamma_curve_x_axis_pnts xa_pnts; +} __packed; + +/** + * struct rkisp1_cif_isp_lsc_config - Configuration used by Lens shading correction + * + * refer to REF_01 for details + */ +struct rkisp1_cif_isp_lsc_config { + __u32 r_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE]; + __u32 gr_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE]; + __u32 gb_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE]; + __u32 b_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE]; + + __u32 x_grad_tbl[RKISP1_CIF_ISP_LSC_GRAD_TBL_SIZE]; + __u32 y_grad_tbl[RKISP1_CIF_ISP_LSC_GRAD_TBL_SIZE]; + + __u32 x_size_tbl[RKISP1_CIF_ISP_LSC_SIZE_TBL_SIZE]; + __u32 y_size_tbl[RKISP1_CIF_ISP_LSC_SIZE_TBL_SIZE]; + __u16 config_width; + __u16 config_height; +} __packed; + +/** + * struct rkisp1_cif_isp_ie_config - Configuration used by image effects + * + * @eff_mat_1: 3x3 Matrix Coefficients for Emboss Effect 1 + * @eff_mat_2: 3x3 Matrix Coefficients for Emboss Effect 2 + * @eff_mat_3: 3x3 Matrix Coefficients for Emboss 3/Sketch 1 + * @eff_mat_4: 3x3 Matrix Coefficients for Sketch Effect 2 + * @eff_mat_5: 3x3 Matrix Coefficients for Sketch Effect 3 + * @eff_tint: Chrominance increment values of tint (used for sepia effect) + */ +struct rkisp1_cif_isp_ie_config { + __u16 effect; + __u16 color_sel; + __u16 eff_mat_1; + __u16 eff_mat_2; + __u16 eff_mat_3; + __u16 eff_mat_4; + __u16 eff_mat_5; + __u16 eff_tint; +} __packed; + +/** + * struct rkisp1_cif_isp_cproc_config - Configuration used by Color Processing + * + * @c_out_range: Chrominance pixel clipping range at output. + * (0 for limit, 1 for full) + * @y_in_range: Luminance pixel clipping range at output. + * @y_out_range: Luminance pixel clipping range at output. + * @contrast: 00~ff, 0.0~1.992 + * @brightness: 80~7F, -128~+127 + * @sat: saturation, 00~FF, 0.0~1.992 + * @hue: 80~7F, -90~+87.188 + */ +struct rkisp1_cif_isp_cproc_config { + __u8 c_out_range; + __u8 y_in_range; + __u8 y_out_range; + __u8 contrast; + __u8 brightness; + __u8 sat; + __u8 hue; +} __packed; + +/** + * struct rkisp1_cif_isp_awb_meas_config - Configuration used by auto white balance + * + * @awb_wnd: white balance measurement window (in pixels) + * (from enum rkisp1_cif_isp_awb_mode_type) + * @max_y: only pixels values < max_y contribute to awb measurement, set to 0 + * to disable this feature + * @min_y: only pixels values > min_y contribute to awb measurement + * @max_csum: Chrominance sum maximum value, only consider pixels with Cb+Cr, + * smaller than threshold for awb measurements + * @min_c: Chrominance minimum value, only consider pixels with Cb/Cr + * each greater than threshold value for awb measurements + * @frames: number of frames - 1 used for mean value calculation + * (ucFrames=0 means 1 Frame) + * @awb_ref_cr: reference Cr value for AWB regulation, target for AWB + * @awb_ref_cb: reference Cb value for AWB regulation, target for AWB + */ +struct rkisp1_cif_isp_awb_meas_config { + /* + * Note: currently the h and v offsets are mapped to grid offsets + */ + struct rkisp1_cif_isp_window awb_wnd; + __u32 awb_mode; + __u8 max_y; + __u8 min_y; + __u8 max_csum; + __u8 min_c; + __u8 frames; + __u8 awb_ref_cr; + __u8 awb_ref_cb; + __u8 enable_ymax_cmp; +} __packed; + +/** + * struct rkisp1_cif_isp_awb_gain_config - Configuration used by auto white balance gain + * + * out_data_x = ( AWB_GEAIN_X * in_data + 128) >> 8 + */ +struct rkisp1_cif_isp_awb_gain_config { + __u16 gain_red; + __u16 gain_green_r; + __u16 gain_blue; + __u16 gain_green_b; +} __packed; + +/** + * struct rkisp1_cif_isp_flt_config - Configuration used by ISP filtering + * + * @mode: ISP_FILT_MODE register fields (from enum rkisp1_cif_isp_flt_mode) + * @grn_stage1: ISP_FILT_MODE register fields + * @chr_h_mode: ISP_FILT_MODE register fields + * @chr_v_mode: ISP_FILT_MODE register fields + * + * refer to REF_01 for details. + */ + +struct rkisp1_cif_isp_flt_config { + __u32 mode; + __u8 grn_stage1; + __u8 chr_h_mode; + __u8 chr_v_mode; + __u32 thresh_bl0; + __u32 thresh_bl1; + __u32 thresh_sh0; + __u32 thresh_sh1; + __u32 lum_weight; + __u32 fac_sh1; + __u32 fac_sh0; + __u32 fac_mid; + __u32 fac_bl0; + __u32 fac_bl1; +} __packed; + +/** + * struct rkisp1_cif_isp_bdm_config - Configuration used by Bayer DeMosaic + * + * @demosaic_th: threshod for bayer demosaicing texture detection + */ +struct rkisp1_cif_isp_bdm_config { + __u8 demosaic_th; +} __packed; + +/** + * struct rkisp1_cif_isp_ctk_config - Configuration used by Cross Talk correction + * + * @coeff: color correction matrix + * @ct_offset_b: offset for the crosstalk correction matrix + */ +struct rkisp1_cif_isp_ctk_config { + __u16 coeff0; + __u16 coeff1; + __u16 coeff2; + __u16 coeff3; + __u16 coeff4; + __u16 coeff5; + __u16 coeff6; + __u16 coeff7; + __u16 coeff8; + __u16 ct_offset_r; + __u16 ct_offset_g; + __u16 ct_offset_b; +} __packed; + +enum rkisp1_cif_isp_goc_mode { + RKISP1_CIF_ISP_GOC_MODE_LOGARITHMIC, + RKISP1_CIF_ISP_GOC_MODE_EQUIDISTANT +}; + +/** + * struct rkisp1_cif_isp_goc_config - Configuration used by Gamma Out correction + * + * @mode: goc mode (from enum rkisp1_cif_isp_goc_mode) + * @gamma_y: gamma out curve y-axis for all color components + */ +struct rkisp1_cif_isp_goc_config { + __u32 mode; + __u16 gamma_y[RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES]; +} __packed; + +/** + * struct rkisp1_cif_isp_hst_config - Configuration used by Histogram + * + * @mode: histogram mode (from enum rkisp1_cif_isp_histogram_mode) + * @histogram_predivider: process every stepsize pixel, all other pixels are + * skipped + * @meas_window: coordinates of the measure window + * @hist_weight: weighting factor for sub-windows + */ +struct rkisp1_cif_isp_hst_config { + __u32 mode; + __u8 histogram_predivider; + struct rkisp1_cif_isp_window meas_window; + __u8 hist_weight[RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE]; +} __packed; + +/** + * struct rkisp1_cif_isp_aec_config - Configuration used by Auto Exposure Control + * + * @mode: Exposure measure mode (from enum rkisp1_cif_isp_exp_meas_mode) + * @autostop: stop mode (from enum rkisp1_cif_isp_exp_ctrl_autostop) + * @meas_window: coordinates of the measure window + */ +struct rkisp1_cif_isp_aec_config { + __u32 mode; + __u32 autostop; + struct rkisp1_cif_isp_window meas_window; +} __packed; + +/** + * struct rkisp1_cif_isp_afc_config - Configuration used by Auto Focus Control + * + * @num_afm_win: max RKISP1_CIF_ISP_AFM_MAX_WINDOWS + * @afm_win: coordinates of the meas window + * @thres: threshold used for minimizing the influence of noise + * @var_shift: the number of bits for the shift operation at the end of the + * calculation chain. + */ +struct rkisp1_cif_isp_afc_config { + __u8 num_afm_win; + struct rkisp1_cif_isp_window afm_win[RKISP1_CIF_ISP_AFM_MAX_WINDOWS]; + __u32 thres; + __u32 var_shift; +} __packed; + +/** + * enum rkisp1_cif_isp_dpf_gain_usage - dpf gain usage + * @RKISP1_CIF_ISP_DPF_GAIN_USAGE_DISABLED: don't use any gains in preprocessing stage + * @RKISP1_CIF_ISP_DPF_GAIN_USAGE_NF_GAINS: use only the noise function gains from + * registers DPF_NF_GAIN_R, ... + * @RKISP1_CIF_ISP_DPF_GAIN_USAGE_LSC_GAINS: use only the gains from LSC module + * @RKISP1_CIF_ISP_DPF_GAIN_USAGE_NF_LSC_GAINS: use the noise function gains and the + * gains from LSC module + * @RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_GAINS: use only the gains from AWB module + * @RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_LSC_GAINS: use the gains from AWB and LSC module + * @RKISP1_CIF_ISP_DPF_GAIN_USAGE_MAX: upper border (only for an internal evaluation) + */ +enum rkisp1_cif_isp_dpf_gain_usage { + RKISP1_CIF_ISP_DPF_GAIN_USAGE_DISABLED, + RKISP1_CIF_ISP_DPF_GAIN_USAGE_NF_GAINS, + RKISP1_CIF_ISP_DPF_GAIN_USAGE_LSC_GAINS, + RKISP1_CIF_ISP_DPF_GAIN_USAGE_NF_LSC_GAINS, + RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_GAINS, + RKISP1_CIF_ISP_DPF_GAIN_USAGE_AWB_LSC_GAINS, + RKISP1_CIF_ISP_DPF_GAIN_USAGE_MAX +}; + +/** + * enum rkisp1_cif_isp_dpf_rb_filtersize - Red and blue filter sizes + * @RKISP1_CIF_ISP_DPF_RB_FILTERSIZE_13x9: red and blue filter kernel size 13x9 + * (means 7x5 active pixel) + * @RKISP1_CIF_ISP_DPF_RB_FILTERSIZE_9x9: red and blue filter kernel size 9x9 + * (means 5x5 active pixel) + */ +enum rkisp1_cif_isp_dpf_rb_filtersize { + RKISP1_CIF_ISP_DPF_RB_FILTERSIZE_13x9, + RKISP1_CIF_ISP_DPF_RB_FILTERSIZE_9x9, +}; + +/** + * enum rkisp1_cif_isp_dpf_nll_scale_mode - dpf noise level scale mode + * @RKISP1_CIF_ISP_NLL_SCALE_LINEAR: use a linear scaling + * @RKISP1_CIF_ISP_NLL_SCALE_LOGARITHMIC: use a logarithmic scaling + */ +enum rkisp1_cif_isp_dpf_nll_scale_mode { + RKISP1_CIF_ISP_NLL_SCALE_LINEAR, + RKISP1_CIF_ISP_NLL_SCALE_LOGARITHMIC, +}; + +/** + * struct rkisp1_cif_isp_dpf_nll - Noise level lookup + * + * @coeff: Noise level Lookup coefficient + * @scale_mode: dpf noise level scale mode (from enum rkisp1_cif_isp_dpf_nll_scale_mode) + */ +struct rkisp1_cif_isp_dpf_nll { + __u16 coeff[RKISP1_CIF_ISP_DPF_MAX_NLF_COEFFS]; + __u32 scale_mode; +} __packed; + +/** + * struct rkisp1_cif_isp_dpf_rb_flt - Red blue filter config + * + * @fltsize: The filter size for the red and blue pixels + * (from enum rkisp1_cif_isp_dpf_rb_filtersize) + * @spatial_coeff: Spatial weights + * @r_enable: enable filter processing for red pixels + * @b_enable: enable filter processing for blue pixels + */ +struct rkisp1_cif_isp_dpf_rb_flt { + __u32 fltsize; + __u8 spatial_coeff[RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 r_enable; + __u8 b_enable; +} __packed; + +/** + * struct rkisp1_cif_isp_dpf_g_flt - Green filter Configuration + * + * @spatial_coeff: Spatial weights + * @gr_enable: enable filter processing for green pixels in green/red lines + * @gb_enable: enable filter processing for green pixels in green/blue lines + */ +struct rkisp1_cif_isp_dpf_g_flt { + __u8 spatial_coeff[RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS]; + __u8 gr_enable; + __u8 gb_enable; +} __packed; + +/** + * struct rkisp1_cif_isp_dpf_gain - Noise function Configuration + * + * @mode: dpf gain usage (from enum rkisp1_cif_isp_dpf_gain_usage) + * @nf_r_gain: Noise function Gain that replaces the AWB gain for red pixels + * @nf_b_gain: Noise function Gain that replaces the AWB gain for blue pixels + * @nf_gr_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a red line + * @nf_gb_gain: Noise function Gain that replaces the AWB gain + * for green pixels in a blue line + */ +struct rkisp1_cif_isp_dpf_gain { + __u32 mode; + __u16 nf_r_gain; + __u16 nf_b_gain; + __u16 nf_gr_gain; + __u16 nf_gb_gain; +} __packed; + +/** + * struct rkisp1_cif_isp_dpf_config - Configuration used by De-noising pre-filter + * + * @gain: noise function gain + * @g_flt: green filter config + * @rb_flt: red blue filter config + * @nll: noise level lookup + */ +struct rkisp1_cif_isp_dpf_config { + struct rkisp1_cif_isp_dpf_gain gain; + struct rkisp1_cif_isp_dpf_g_flt g_flt; + struct rkisp1_cif_isp_dpf_rb_flt rb_flt; + struct rkisp1_cif_isp_dpf_nll nll; +} __packed; + +/** + * struct rkisp1_cif_isp_dpf_strength_config - strength of the filter + * + * @r: filter strength of the RED filter + * @g: filter strength of the GREEN filter + * @b: filter strength of the BLUE filter + */ +struct rkisp1_cif_isp_dpf_strength_config { + __u8 r; + __u8 g; + __u8 b; +} __packed; + +/** + * struct rkisp1_cif_isp_isp_other_cfg - Parameters for some blocks in rockchip isp1 + * + * @dpcc_config: Defect Pixel Cluster Correction config + * @bls_config: Black Level Subtraction config + * @sdg_config: sensor degamma config + * @lsc_config: Lens Shade config + * @awb_gain_config: Auto White balance gain config + * @flt_config: filter config + * @bdm_config: demosaic config + * @ctk_config: cross talk config + * @goc_config: gamma out config + * @bls_config: black level subtraction config + * @dpf_config: De-noising pre-filter config + * @dpf_strength_config: dpf strength config + * @cproc_config: color process config + * @ie_config: image effects config + */ +struct rkisp1_cif_isp_isp_other_cfg { + struct rkisp1_cif_isp_dpcc_config dpcc_config; + struct rkisp1_cif_isp_bls_config bls_config; + struct rkisp1_cif_isp_sdg_config sdg_config; + struct rkisp1_cif_isp_lsc_config lsc_config; + struct rkisp1_cif_isp_awb_gain_config awb_gain_config; + struct rkisp1_cif_isp_flt_config flt_config; + struct rkisp1_cif_isp_bdm_config bdm_config; + struct rkisp1_cif_isp_ctk_config ctk_config; + struct rkisp1_cif_isp_goc_config goc_config; + struct rkisp1_cif_isp_dpf_config dpf_config; + struct rkisp1_cif_isp_dpf_strength_config dpf_strength_config; + struct rkisp1_cif_isp_cproc_config cproc_config; + struct rkisp1_cif_isp_ie_config ie_config; +} __packed; + +/** + * struct rkisp1_cif_isp_isp_meas_cfg - Rockchip ISP1 Measure Parameters + * + * @awb_meas_config: auto white balance config + * @hst_config: histogram config + * @aec_config: auto exposure config + * @afc_config: auto focus config + */ +struct rkisp1_cif_isp_isp_meas_cfg { + struct rkisp1_cif_isp_awb_meas_config awb_meas_config; + struct rkisp1_cif_isp_hst_config hst_config; + struct rkisp1_cif_isp_aec_config aec_config; + struct rkisp1_cif_isp_afc_config afc_config; +} __packed; + +/** + * struct rkisp1_params_cfg - Rockchip ISP1 Input Parameters Meta Data + * + * @module_en_update: mask the enable bits of which module should be updated + * @module_ens: mask the enable value of each module, only update the module + * which correspond bit was set in module_en_update + * @module_cfg_update: mask the config bits of which module should be updated + * @meas: measurement config + * @others: other config + */ +struct rkisp1_params_cfg { + __u32 module_en_update; + __u32 module_ens; + __u32 module_cfg_update; + + struct rkisp1_cif_isp_isp_meas_cfg meas; + struct rkisp1_cif_isp_isp_other_cfg others; +} __packed; + +/*---------- PART2: Measurement Statistics ------------*/ + +/** + * struct rkisp1_cif_isp_awb_meas - AWB measured values + * + * @cnt: White pixel count, number of "white pixels" found during last + * measurement + * @mean_y_or_g: Mean value of Y within window and frames, + * Green if RGB is selected. + * @mean_cb_or_b: Mean value of Cb within window and frames, + * Blue if RGB is selected. + * @mean_cr_or_r: Mean value of Cr within window and frames, + * Red if RGB is selected. + */ +struct rkisp1_cif_isp_awb_meas { + __u32 cnt; + __u8 mean_y_or_g; + __u8 mean_cb_or_b; + __u8 mean_cr_or_r; +} __packed; + +/** + * struct rkisp1_cif_isp_awb_stat - statistics automatic white balance data + * + * @awb_mean: Mean measured data + */ +struct rkisp1_cif_isp_awb_stat { + struct rkisp1_cif_isp_awb_meas awb_mean[RKISP1_CIF_ISP_AWB_MAX_GRID]; +} __packed; + +/** + * struct rkisp1_cif_isp_bls_meas_val - BLS measured values + * + * @meas_r: Mean measured value for Bayer pattern R + * @meas_gr: Mean measured value for Bayer pattern Gr + * @meas_gb: Mean measured value for Bayer pattern Gb + * @meas_b: Mean measured value for Bayer pattern B + */ +struct rkisp1_cif_isp_bls_meas_val { + __u16 meas_r; + __u16 meas_gr; + __u16 meas_gb; + __u16 meas_b; +} __packed; + +/** + * struct rkisp1_cif_isp_ae_stat - statistics auto exposure data + * + * @exp_mean: Mean luminance value of block xx + * @bls_val: BLS measured values + * + * Image is divided into 5x5 blocks. + */ +struct rkisp1_cif_isp_ae_stat { + __u8 exp_mean[RKISP1_CIF_ISP_AE_MEAN_MAX]; + struct rkisp1_cif_isp_bls_meas_val bls_val; +} __packed; + +/** + * struct rkisp1_cif_isp_af_meas_val - AF measured values + * + * @sum: sharpness, refer to REF_01 for definition + * @lum: luminance, refer to REF_01 for definition + */ +struct rkisp1_cif_isp_af_meas_val { + __u32 sum; + __u32 lum; +} __packed; + +/** + * struct rkisp1_cif_isp_af_stat - statistics auto focus data + * + * @window: AF measured value of window x + * + * The module measures the sharpness in 3 windows of selectable size via + * register settings(ISP_AFM_*_A/B/C) + */ +struct rkisp1_cif_isp_af_stat { + struct rkisp1_cif_isp_af_meas_val window[RKISP1_CIF_ISP_AFM_MAX_WINDOWS]; +} __packed; + +/** + * struct rkisp1_cif_isp_hist_stat - statistics histogram data + * + * @hist_bins: measured bin counters + * + * Measurement window divided into 25 sub-windows, set + * with ISP_HIST_XXX + */ +struct rkisp1_cif_isp_hist_stat { + __u16 hist_bins[RKISP1_CIF_ISP_HIST_BIN_N_MAX]; +} __packed; + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Data + * + * @rkisp1_cif_isp_awb_stat: statistics data for automatic white balance + * @rkisp1_cif_isp_ae_stat: statistics data for auto exposure + * @rkisp1_cif_isp_af_stat: statistics data for auto focus + * @rkisp1_cif_isp_hist_stat: statistics histogram data + */ +struct rkisp1_cif_isp_stat { + struct rkisp1_cif_isp_awb_stat awb; + struct rkisp1_cif_isp_ae_stat ae; + struct rkisp1_cif_isp_af_stat af; + struct rkisp1_cif_isp_hist_stat hist; +} __packed; + +/** + * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Meta Data + * + * @meas_type: measurement types (RKISP1_CIF_ISP_STAT_ definitions) + * @frame_id: frame ID for sync + * @params: statistics data + */ +struct rkisp1_stat_buffer { + __u32 meas_type; + __u32 frame_id; + struct rkisp1_cif_isp_stat params; +} __packed; + +#endif /* _UAPI_RKISP1_CONFIG_H */ diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c index 6945dc74e1d7..ce497d0197df 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c @@ -7,6 +7,7 @@ * Copyright (C) 2018 Bootlin */ +#include <linux/delay.h> #include <linux/types.h> #include <media/videobuf2-dma-contig.h> @@ -220,6 +221,23 @@ static void cedrus_h265_pred_weight_write(struct cedrus_dev *dev, } } +static void cedrus_h265_skip_bits(struct cedrus_dev *dev, int num) +{ + int count = 0; + + while (count < num) { + int tmp = min(num - count, 32); + + cedrus_write(dev, VE_DEC_H265_TRIGGER, + VE_DEC_H265_TRIGGER_FLUSH_BITS | + VE_DEC_H265_TRIGGER_TYPE_N_BITS(tmp)); + while (cedrus_read(dev, VE_DEC_H265_STATUS) & VE_DEC_H265_STATUS_VLD_BUSY) + udelay(1); + + count += tmp; + } +} + static void cedrus_h265_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) { @@ -280,10 +298,9 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, /* Source offset and length in bits. */ - reg = slice_params->data_bit_offset; - cedrus_write(dev, VE_DEC_H265_BITS_OFFSET, reg); + cedrus_write(dev, VE_DEC_H265_BITS_OFFSET, 0); - reg = slice_params->bit_size - slice_params->data_bit_offset; + reg = slice_params->bit_size; cedrus_write(dev, VE_DEC_H265_BITS_LEN, reg); /* Source beginning and end addresses. */ @@ -316,6 +333,8 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, /* Initialize bitstream access. */ cedrus_write(dev, VE_DEC_H265_TRIGGER, VE_DEC_H265_TRIGGER_INIT_SWDEC); + cedrus_h265_skip_bits(dev, slice_params->data_bit_offset); + /* Bitstream parameters. */ reg = VE_DEC_H265_DEC_NAL_HDR_NAL_UNIT_TYPE(slice_params->nal_unit_type) | @@ -332,6 +351,7 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, VE_DEC_H265_DEC_SPS_HDR_LOG2_DIFF_MAX_MIN_LUMA_CODING_BLOCK_SIZE(sps->log2_diff_max_min_luma_coding_block_size) | VE_DEC_H265_DEC_SPS_HDR_LOG2_MIN_LUMA_CODING_BLOCK_SIZE_MINUS3(sps->log2_min_luma_coding_block_size_minus3) | VE_DEC_H265_DEC_SPS_HDR_BIT_DEPTH_CHROMA_MINUS8(sps->bit_depth_chroma_minus8) | + VE_DEC_H265_DEC_SPS_HDR_BIT_DEPTH_LUMA_MINUS8(sps->bit_depth_luma_minus8) | VE_DEC_H265_DEC_SPS_HDR_CHROMA_FORMAT_IDC(sps->chroma_format_idc); reg |= VE_DEC_H265_FLAG(VE_DEC_H265_DEC_SPS_HDR_FLAG_STRONG_INTRA_SMOOTHING_ENABLE, diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_regs.h b/drivers/staging/media/sunxi/cedrus/cedrus_regs.h index 7beb03d3bb39..66b152f18d17 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_regs.h +++ b/drivers/staging/media/sunxi/cedrus/cedrus_regs.h @@ -424,6 +424,7 @@ #define VE_DEC_H265_TRIGGER (VE_ENGINE_DEC_H265 + 0x34) +#define VE_DEC_H265_TRIGGER_TYPE_N_BITS(x) (((x) & 0x3f) << 8) #define VE_DEC_H265_TRIGGER_STCD_VC1 (0x02 << 4) #define VE_DEC_H265_TRIGGER_STCD_AVS (0x01 << 4) #define VE_DEC_H265_TRIGGER_STCD_HEVC (0x00 << 4) diff --git a/drivers/staging/media/tegra-vde/Kconfig b/drivers/staging/media/tegra-vde/Kconfig index ba49ea50b8c0..0dc78afd09e0 100644 --- a/drivers/staging/media/tegra-vde/Kconfig +++ b/drivers/staging/media/tegra-vde/Kconfig @@ -3,7 +3,7 @@ config TEGRA_VDE tristate "NVIDIA Tegra Video Decoder Engine driver" depends on ARCH_TEGRA || COMPILE_TEST select DMA_SHARED_BUFFER - select IOMMU_IOVA if (IOMMU_SUPPORT || COMPILE_TEST) + select IOMMU_IOVA select SRAM help Say Y here to enable support for the NVIDIA Tegra video decoder diff --git a/drivers/staging/media/tegra-vde/vde.c b/drivers/staging/media/tegra-vde/vde.c index 3466daddf663..e18fd48981da 100644 --- a/drivers/staging/media/tegra-vde/vde.c +++ b/drivers/staging/media/tegra-vde/vde.c @@ -1150,8 +1150,7 @@ static int tegra_vde_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM_SLEEP -static int tegra_vde_pm_suspend(struct device *dev) +static __maybe_unused int tegra_vde_pm_suspend(struct device *dev) { struct tegra_vde *vde = dev_get_drvdata(dev); int err; @@ -1165,7 +1164,7 @@ static int tegra_vde_pm_suspend(struct device *dev) return 0; } -static int tegra_vde_pm_resume(struct device *dev) +static __maybe_unused int tegra_vde_pm_resume(struct device *dev) { struct tegra_vde *vde = dev_get_drvdata(dev); int err; @@ -1178,7 +1177,6 @@ static int tegra_vde_pm_resume(struct device *dev) return 0; } -#endif static const struct dev_pm_ops tegra_vde_pm_ops = { SET_RUNTIME_PM_OPS(tegra_vde_runtime_suspend, diff --git a/drivers/staging/media/tegra-vde/vde.h b/drivers/staging/media/tegra-vde/vde.h index d369f1466bc7..5561291b0c88 100644 --- a/drivers/staging/media/tegra-vde/vde.h +++ b/drivers/staging/media/tegra-vde/vde.h @@ -10,11 +10,11 @@ #include <linux/completion.h> #include <linux/dma-direction.h> +#include <linux/iova.h> #include <linux/list.h> #include <linux/miscdevice.h> #include <linux/mutex.h> #include <linux/types.h> -#include <linux/iova.h> struct clk; struct dma_buf; diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_module.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_module.c index 4a6c3f674431..aa9dab8a4ae8 100644 --- a/drivers/staging/rtl8192u/ieee80211/ieee80211_module.c +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_module.c @@ -264,12 +264,12 @@ static int open_debug_level(struct inode *inode, struct file *file) return single_open(file, show_debug_level, NULL); } -static const struct file_operations fops = { - .open = open_debug_level, - .read = seq_read, - .llseek = seq_lseek, - .write = write_debug_level, - .release = single_release, +static const struct proc_ops debug_level_proc_ops = { + .proc_open = open_debug_level, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_write = write_debug_level, + .proc_release = single_release, }; int __init ieee80211_debug_init(void) @@ -284,7 +284,7 @@ int __init ieee80211_debug_init(void) " proc directory\n"); return -EIO; } - e = proc_create("debug_level", 0644, ieee80211_proc, &fops); + e = proc_create("debug_level", 0644, ieee80211_proc, &debug_level_proc_ops); if (!e) { remove_proc_entry(DRV_NAME, init_net.proc_net); ieee80211_proc = NULL; diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c index 1cc5e6c5709e..ad9e3bf8fdf6 100644 --- a/drivers/thermal/st/stm_thermal.c +++ b/drivers/thermal/st/stm_thermal.c @@ -535,7 +535,7 @@ static int stm_thermal_probe(struct platform_device *pdev) /* Configure and enable HW sensor */ ret = stm_thermal_prepare(sensor); if (ret) { - dev_err(&pdev->dev, "Error preprare sensor: %d\n", ret); + dev_err(&pdev->dev, "Error prepare sensor: %d\n", ret); return ret; } diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c index 1d4f317a0e42..f724962a5906 100644 --- a/drivers/tty/sysrq.c +++ b/drivers/tty/sysrq.c @@ -1101,15 +1101,15 @@ static ssize_t write_sysrq_trigger(struct file *file, const char __user *buf, return count; } -static const struct file_operations proc_sysrq_trigger_operations = { - .write = write_sysrq_trigger, - .llseek = noop_llseek, +static const struct proc_ops sysrq_trigger_proc_ops = { + .proc_write = write_sysrq_trigger, + .proc_lseek = noop_llseek, }; static void sysrq_init_procfs(void) { if (!proc_create("sysrq-trigger", S_IWUSR, NULL, - &proc_sysrq_trigger_operations)) + &sysrq_trigger_proc_ops)) pr_err("Failed to register proc interface\n"); } diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c index 04c142c13075..64de9f1b874c 100644 --- a/drivers/usb/gadget/function/rndis.c +++ b/drivers/usb/gadget/function/rndis.c @@ -72,7 +72,7 @@ static rndis_resp_t *rndis_add_response(struct rndis_params *params, #ifdef CONFIG_USB_GADGET_DEBUG_FILES -static const struct file_operations rndis_proc_fops; +static const struct proc_ops rndis_proc_ops; #endif /* CONFIG_USB_GADGET_DEBUG_FILES */ @@ -902,7 +902,7 @@ struct rndis_params *rndis_register(void (*resp_avail)(void *v), void *v) sprintf(name, NAME_TEMPLATE, i); proc_entry = proc_create_data(name, 0660, NULL, - &rndis_proc_fops, params); + &rndis_proc_ops, params); if (!proc_entry) { kfree(params); rndis_put_nr(i); @@ -1164,13 +1164,12 @@ static int rndis_proc_open(struct inode *inode, struct file *file) return single_open(file, rndis_proc_show, PDE_DATA(inode)); } -static const struct file_operations rndis_proc_fops = { - .owner = THIS_MODULE, - .open = rndis_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = rndis_proc_write, +static const struct proc_ops rndis_proc_ops = { + .proc_open = rndis_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = rndis_proc_write, }; #define NAME_TEMPLATE "driver/rndis-%03d" diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c index 7570c7602ab4..8ad14e5c02bf 100644 --- a/drivers/vfio/mdev/mdev_sysfs.c +++ b/drivers/vfio/mdev/mdev_sysfs.c @@ -74,7 +74,7 @@ static ssize_t create_store(struct kobject *kobj, struct device *dev, return count; } -MDEV_TYPE_ATTR_WO(create); +static MDEV_TYPE_ATTR_WO(create); static void mdev_type_release(struct kobject *kobj) { diff --git a/drivers/vfio/pci/vfio_pci_nvlink2.c b/drivers/vfio/pci/vfio_pci_nvlink2.c index f2983f0f84be..df4d96038cd4 100644 --- a/drivers/vfio/pci/vfio_pci_nvlink2.c +++ b/drivers/vfio/pci/vfio_pci_nvlink2.c @@ -97,8 +97,10 @@ static void vfio_pci_nvgpu_release(struct vfio_pci_device *vdev, /* If there were any mappings at all... */ if (data->mm) { - ret = mm_iommu_put(data->mm, data->mem); - WARN_ON(ret); + if (data->mem) { + ret = mm_iommu_put(data->mm, data->mem); + WARN_ON(ret); + } mmdrop(data->mm); } @@ -159,7 +161,7 @@ static int vfio_pci_nvgpu_mmap(struct vfio_pci_device *vdev, data->useraddr = vma->vm_start; data->mm = current->mm; - atomic_inc(&data->mm->mm_count); + mmgrab(data->mm); ret = (int) mm_iommu_newdev(data->mm, data->useraddr, vma_pages(vma), data->gpu_hpa, &data->mem); diff --git a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c index 40d4fb9276ba..abdca900802d 100644 --- a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c +++ b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c @@ -24,7 +24,7 @@ #define MDIO_AN_INT 0x8002 #define MDIO_AN_INTMASK 0x8001 -static unsigned int xmdio_read(void *ioaddr, unsigned int mmd, +static unsigned int xmdio_read(void __iomem *ioaddr, unsigned int mmd, unsigned int reg) { unsigned int mmd_address, value; @@ -35,7 +35,7 @@ static unsigned int xmdio_read(void *ioaddr, unsigned int mmd, return value; } -static void xmdio_write(void *ioaddr, unsigned int mmd, +static void xmdio_write(void __iomem *ioaddr, unsigned int mmd, unsigned int reg, unsigned int value) { unsigned int mmd_address; diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c index 26cef65b41e7..16b3adc508db 100644 --- a/drivers/vfio/vfio_iommu_spapr_tce.c +++ b/drivers/vfio/vfio_iommu_spapr_tce.c @@ -79,7 +79,7 @@ static long tce_iommu_mm_set(struct tce_container *container) } BUG_ON(!current->mm); container->mm = current->mm; - atomic_inc(&container->mm->mm_count); + mmgrab(container->mm); return 0; } diff --git a/drivers/video/backlight/ams369fg06.c b/drivers/video/backlight/ams369fg06.c index 94ccb9042440..8a4361e95a11 100644 --- a/drivers/video/backlight/ams369fg06.c +++ b/drivers/video/backlight/ams369fg06.c @@ -11,7 +11,6 @@ #include <linux/backlight.h> #include <linux/delay.h> #include <linux/fb.h> -#include <linux/gpio.h> #include <linux/lcd.h> #include <linux/module.h> #include <linux/spi/spi.h> diff --git a/drivers/video/backlight/bd6107.c b/drivers/video/backlight/bd6107.c index d344fb03cb86..d5d5fb457e78 100644 --- a/drivers/video/backlight/bd6107.c +++ b/drivers/video/backlight/bd6107.c @@ -11,7 +11,7 @@ #include <linux/delay.h> #include <linux/err.h> #include <linux/fb.h> -#include <linux/gpio.h> +#include <linux/gpio/consumer.h> #include <linux/i2c.h> #include <linux/module.h> #include <linux/platform_data/bd6107.h> @@ -71,6 +71,7 @@ struct bd6107 { struct i2c_client *client; struct backlight_device *backlight; struct bd6107_platform_data *pdata; + struct gpio_desc *reset; }; static int bd6107_write(struct bd6107 *bd, u8 reg, u8 data) @@ -94,9 +95,10 @@ static int bd6107_backlight_update_status(struct backlight_device *backlight) bd6107_write(bd, BD6107_MAINCNT1, brightness); bd6107_write(bd, BD6107_LEDCNT1, BD6107_LEDCNT1_LEDONOFF1); } else { - gpio_set_value(bd->pdata->reset, 0); + /* Assert the reset line (gpiolib will handle active low) */ + gpiod_set_value(bd->reset, 1); msleep(24); - gpio_set_value(bd->pdata->reset, 1); + gpiod_set_value(bd->reset, 0); } return 0; @@ -125,8 +127,8 @@ static int bd6107_probe(struct i2c_client *client, struct bd6107 *bd; int ret; - if (pdata == NULL || !pdata->reset) { - dev_err(&client->dev, "No reset GPIO in platform data\n"); + if (pdata == NULL) { + dev_err(&client->dev, "No platform data\n"); return -EINVAL; } @@ -144,10 +146,16 @@ static int bd6107_probe(struct i2c_client *client, bd->client = client; bd->pdata = pdata; - ret = devm_gpio_request_one(&client->dev, pdata->reset, - GPIOF_DIR_OUT | GPIOF_INIT_LOW, "reset"); - if (ret < 0) { + /* + * Request the reset GPIO line with GPIOD_OUT_HIGH meaning asserted, + * so in the machine descriptor table (or other hardware description), + * the line should be flagged as active low so this will assert + * the reset. + */ + bd->reset = devm_gpiod_get(&client->dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(bd->reset)) { dev_err(&client->dev, "unable to request reset GPIO\n"); + ret = PTR_ERR(bd->reset); return ret; } diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c index d46052d8ff41..3d276b30a78c 100644 --- a/drivers/video/backlight/qcom-wled.c +++ b/drivers/video/backlight/qcom-wled.c @@ -956,8 +956,8 @@ static int wled_configure(struct wled *wled, int version) struct wled_config *cfg = &wled->cfg; struct device *dev = wled->dev; const __be32 *prop_addr; - u32 size, val, c, string_len; - int rc, i, j; + u32 size, val, c; + int rc, i, j, string_len; const struct wled_u32_opts *u32_opts = NULL; const struct wled_u32_opts wled3_opts[] = { diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig index aa9541bf964b..f65991a67af2 100644 --- a/drivers/video/fbdev/Kconfig +++ b/drivers/video/fbdev/Kconfig @@ -2215,6 +2215,7 @@ config FB_HYPERV select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT select FB_DEFERRED_IO + select DMA_CMA if HAVE_DMA_CONTIGUOUS && CMA help This framebuffer driver supports Microsoft Hyper-V Synthetic Video. diff --git a/drivers/video/fbdev/hyperv_fb.c b/drivers/video/fbdev/hyperv_fb.c index afe9fd751cd5..f47d50e560c0 100644 --- a/drivers/video/fbdev/hyperv_fb.c +++ b/drivers/video/fbdev/hyperv_fb.c @@ -31,6 +31,16 @@ * "set-vmvideo" command. For example * set-vmvideo -vmname name -horizontalresolution:1920 \ * -verticalresolution:1200 -resolutiontype single + * + * Gen 1 VMs also support direct using VM's physical memory for framebuffer. + * It could improve the efficiency and performance for framebuffer and VM. + * This requires to allocate contiguous physical memory from Linux kernel's + * CMA memory allocator. To enable this, supply a kernel parameter to give + * enough memory space to CMA allocator for framebuffer. For example: + * cma=130m + * This gives 130MB memory to CMA allocator that can be allocated to + * framebuffer. For reference, 8K resolution (7680x4320) takes about + * 127MB memory. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -228,7 +238,6 @@ struct synthvid_msg { } __packed; - /* FB driver definitions and structures */ #define HVFB_WIDTH 1152 /* default screen width */ #define HVFB_HEIGHT 864 /* default screen height */ @@ -258,12 +267,15 @@ struct hvfb_par { /* If true, the VSC notifies the VSP on every framebuffer change */ bool synchronous_fb; + /* If true, need to copy from deferred IO mem to framebuffer mem */ + bool need_docopy; + struct notifier_block hvfb_panic_nb; /* Memory for deferred IO and frame buffer itself */ unsigned char *dio_vp; unsigned char *mmio_vp; - unsigned long mmio_pp; + phys_addr_t mmio_pp; /* Dirty rectangle, protected by delayed_refresh_lock */ int x1, y1, x2, y2; @@ -434,7 +446,7 @@ static void synthvid_deferred_io(struct fb_info *p, maxy = max_t(int, maxy, y2); /* Copy from dio space to mmio address */ - if (par->fb_ready) + if (par->fb_ready && par->need_docopy) hvfb_docopy(par, start, PAGE_SIZE); } @@ -751,12 +763,12 @@ static void hvfb_update_work(struct work_struct *w) return; /* Copy the dirty rectangle to frame buffer memory */ - for (j = y1; j < y2; j++) { - hvfb_docopy(par, - j * info->fix.line_length + - (x1 * screen_depth / 8), - (x2 - x1) * screen_depth / 8); - } + if (par->need_docopy) + for (j = y1; j < y2; j++) + hvfb_docopy(par, + j * info->fix.line_length + + (x1 * screen_depth / 8), + (x2 - x1) * screen_depth / 8); /* Refresh */ if (par->fb_ready && par->update) @@ -801,7 +813,8 @@ static int hvfb_on_panic(struct notifier_block *nb, par = container_of(nb, struct hvfb_par, hvfb_panic_nb); par->synchronous_fb = true; info = par->info; - hvfb_docopy(par, 0, dio_fb_size); + if (par->need_docopy) + hvfb_docopy(par, 0, dio_fb_size); synthvid_update(info, 0, 0, INT_MAX, INT_MAX); return NOTIFY_DONE; @@ -940,6 +953,62 @@ static void hvfb_get_option(struct fb_info *info) return; } +/* + * Allocate enough contiguous physical memory. + * Return physical address if succeeded or -1 if failed. + */ +static phys_addr_t hvfb_get_phymem(struct hv_device *hdev, + unsigned int request_size) +{ + struct page *page = NULL; + dma_addr_t dma_handle; + void *vmem; + phys_addr_t paddr = 0; + unsigned int order = get_order(request_size); + + if (request_size == 0) + return -1; + + if (order < MAX_ORDER) { + /* Call alloc_pages if the size is less than 2^MAX_ORDER */ + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!page) + return -1; + + paddr = (page_to_pfn(page) << PAGE_SHIFT); + } else { + /* Allocate from CMA */ + hdev->device.coherent_dma_mask = DMA_BIT_MASK(64); + + vmem = dma_alloc_coherent(&hdev->device, + round_up(request_size, PAGE_SIZE), + &dma_handle, + GFP_KERNEL | __GFP_NOWARN); + + if (!vmem) + return -1; + + paddr = virt_to_phys(vmem); + } + + return paddr; +} + +/* Release contiguous physical memory */ +static void hvfb_release_phymem(struct hv_device *hdev, + phys_addr_t paddr, unsigned int size) +{ + unsigned int order = get_order(size); + + if (order < MAX_ORDER) + __free_pages(pfn_to_page(paddr >> PAGE_SHIFT), order); + else + dma_free_coherent(&hdev->device, + round_up(size, PAGE_SIZE), + phys_to_virt(paddr), + paddr); +} + /* Get framebuffer memory from Hyper-V video pci space */ static int hvfb_getmem(struct hv_device *hdev, struct fb_info *info) @@ -949,22 +1018,61 @@ static int hvfb_getmem(struct hv_device *hdev, struct fb_info *info) void __iomem *fb_virt; int gen2vm = efi_enabled(EFI_BOOT); resource_size_t pot_start, pot_end; + phys_addr_t paddr; int ret; - dio_fb_size = - screen_width * screen_height * screen_depth / 8; + info->apertures = alloc_apertures(1); + if (!info->apertures) + return -ENOMEM; - if (gen2vm) { - pot_start = 0; - pot_end = -1; - } else { + if (!gen2vm) { pdev = pci_get_device(PCI_VENDOR_ID_MICROSOFT, - PCI_DEVICE_ID_HYPERV_VIDEO, NULL); + PCI_DEVICE_ID_HYPERV_VIDEO, NULL); if (!pdev) { pr_err("Unable to find PCI Hyper-V video\n"); + kfree(info->apertures); return -ENODEV; } + info->apertures->ranges[0].base = pci_resource_start(pdev, 0); + info->apertures->ranges[0].size = pci_resource_len(pdev, 0); + + /* + * For Gen 1 VM, we can directly use the contiguous memory + * from VM. If we succeed, deferred IO happens directly + * on this allocated framebuffer memory, avoiding extra + * memory copy. + */ + paddr = hvfb_get_phymem(hdev, screen_fb_size); + if (paddr != (phys_addr_t) -1) { + par->mmio_pp = paddr; + par->mmio_vp = par->dio_vp = __va(paddr); + + info->fix.smem_start = paddr; + info->fix.smem_len = screen_fb_size; + info->screen_base = par->mmio_vp; + info->screen_size = screen_fb_size; + + par->need_docopy = false; + goto getmem_done; + } + pr_info("Unable to allocate enough contiguous physical memory on Gen 1 VM. Using MMIO instead.\n"); + } else { + info->apertures->ranges[0].base = screen_info.lfb_base; + info->apertures->ranges[0].size = screen_info.lfb_size; + } + + /* + * Cannot use the contiguous physical memory. + * Allocate mmio space for framebuffer. + */ + dio_fb_size = + screen_width * screen_height * screen_depth / 8; + + if (gen2vm) { + pot_start = 0; + pot_end = -1; + } else { if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || pci_resource_len(pdev, 0) < screen_fb_size) { pr_err("Resource not available or (0x%lx < 0x%lx)\n", @@ -993,20 +1101,6 @@ static int hvfb_getmem(struct hv_device *hdev, struct fb_info *info) if (par->dio_vp == NULL) goto err3; - info->apertures = alloc_apertures(1); - if (!info->apertures) - goto err4; - - if (gen2vm) { - info->apertures->ranges[0].base = screen_info.lfb_base; - info->apertures->ranges[0].size = screen_info.lfb_size; - remove_conflicting_framebuffers(info->apertures, - KBUILD_MODNAME, false); - } else { - info->apertures->ranges[0].base = pci_resource_start(pdev, 0); - info->apertures->ranges[0].size = pci_resource_len(pdev, 0); - } - /* Physical address of FB device */ par->mmio_pp = par->mem->start; /* Virtual address of FB device */ @@ -1017,13 +1111,15 @@ static int hvfb_getmem(struct hv_device *hdev, struct fb_info *info) info->screen_base = par->dio_vp; info->screen_size = dio_fb_size; +getmem_done: + remove_conflicting_framebuffers(info->apertures, + KBUILD_MODNAME, false); if (!gen2vm) pci_dev_put(pdev); + kfree(info->apertures); return 0; -err4: - vfree(par->dio_vp); err3: iounmap(fb_virt); err2: @@ -1032,18 +1128,25 @@ err2: err1: if (!gen2vm) pci_dev_put(pdev); + kfree(info->apertures); return -ENOMEM; } /* Release the framebuffer */ -static void hvfb_putmem(struct fb_info *info) +static void hvfb_putmem(struct hv_device *hdev, struct fb_info *info) { struct hvfb_par *par = info->par; - vfree(par->dio_vp); - iounmap(info->screen_base); - vmbus_free_mmio(par->mem->start, screen_fb_size); + if (par->need_docopy) { + vfree(par->dio_vp); + iounmap(info->screen_base); + vmbus_free_mmio(par->mem->start, screen_fb_size); + } else { + hvfb_release_phymem(hdev, info->fix.smem_start, + screen_fb_size); + } + par->mem = NULL; } @@ -1062,6 +1165,7 @@ static int hvfb_probe(struct hv_device *hdev, par = info->par; par->info = info; par->fb_ready = false; + par->need_docopy = true; init_completion(&par->wait); INIT_DELAYED_WORK(&par->dwork, hvfb_update_work); @@ -1147,7 +1251,7 @@ static int hvfb_probe(struct hv_device *hdev, error: fb_deferred_io_cleanup(info); - hvfb_putmem(info); + hvfb_putmem(hdev, info); error2: vmbus_close(hdev->channel); error1: @@ -1177,7 +1281,7 @@ static int hvfb_remove(struct hv_device *hdev) vmbus_close(hdev->channel); hv_set_drvdata(hdev, NULL); - hvfb_putmem(info); + hvfb_putmem(hdev, info); framebuffer_release(info); return 0; @@ -1194,6 +1298,7 @@ static int hvfb_suspend(struct hv_device *hdev) fb_set_suspend(info, 1); cancel_delayed_work_sync(&par->dwork); + cancel_delayed_work_sync(&info->deferred_work); par->update_saved = par->update; par->update = false; @@ -1227,6 +1332,7 @@ static int hvfb_resume(struct hv_device *hdev) par->fb_ready = true; par->update = par->update_saved; + schedule_delayed_work(&info->deferred_work, info->fbdefio->delay); schedule_delayed_work(&par->dwork, HVFB_UPDATE_DELAY); /* 0 means do resume */ diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c index 08a17eb2a5c7..370bf2553d43 100644 --- a/drivers/video/fbdev/imxfb.c +++ b/drivers/video/fbdev/imxfb.c @@ -1017,7 +1017,7 @@ static int imxfb_probe(struct platform_device *pdev) } fbi->lcd_pwr = devm_regulator_get(&pdev->dev, "lcd"); - if (IS_ERR(fbi->lcd_pwr) && (PTR_ERR(fbi->lcd_pwr) == -EPROBE_DEFER)) { + if (PTR_ERR(fbi->lcd_pwr) == -EPROBE_DEFER) { ret = -EPROBE_DEFER; goto failed_lcd; } diff --git a/drivers/video/fbdev/via/viafbdev.c b/drivers/video/fbdev/via/viafbdev.c index f815f98190bc..852673c40a2f 100644 --- a/drivers/video/fbdev/via/viafbdev.c +++ b/drivers/video/fbdev/via/viafbdev.c @@ -1173,13 +1173,12 @@ static ssize_t viafb_dvp0_proc_write(struct file *file, return count; } -static const struct file_operations viafb_dvp0_proc_fops = { - .owner = THIS_MODULE, - .open = viafb_dvp0_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = viafb_dvp0_proc_write, +static const struct proc_ops viafb_dvp0_proc_ops = { + .proc_open = viafb_dvp0_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = viafb_dvp0_proc_write, }; static int viafb_dvp1_proc_show(struct seq_file *m, void *v) @@ -1238,13 +1237,12 @@ static ssize_t viafb_dvp1_proc_write(struct file *file, return count; } -static const struct file_operations viafb_dvp1_proc_fops = { - .owner = THIS_MODULE, - .open = viafb_dvp1_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = viafb_dvp1_proc_write, +static const struct proc_ops viafb_dvp1_proc_ops = { + .proc_open = viafb_dvp1_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = viafb_dvp1_proc_write, }; static int viafb_dfph_proc_show(struct seq_file *m, void *v) @@ -1273,13 +1271,12 @@ static ssize_t viafb_dfph_proc_write(struct file *file, return count; } -static const struct file_operations viafb_dfph_proc_fops = { - .owner = THIS_MODULE, - .open = viafb_dfph_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = viafb_dfph_proc_write, +static const struct proc_ops viafb_dfph_proc_ops = { + .proc_open = viafb_dfph_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = viafb_dfph_proc_write, }; static int viafb_dfpl_proc_show(struct seq_file *m, void *v) @@ -1308,13 +1305,12 @@ static ssize_t viafb_dfpl_proc_write(struct file *file, return count; } -static const struct file_operations viafb_dfpl_proc_fops = { - .owner = THIS_MODULE, - .open = viafb_dfpl_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = viafb_dfpl_proc_write, +static const struct proc_ops viafb_dfpl_proc_ops = { + .proc_open = viafb_dfpl_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = viafb_dfpl_proc_write, }; static int viafb_vt1636_proc_show(struct seq_file *m, void *v) @@ -1444,13 +1440,12 @@ static ssize_t viafb_vt1636_proc_write(struct file *file, return count; } -static const struct file_operations viafb_vt1636_proc_fops = { - .owner = THIS_MODULE, - .open = viafb_vt1636_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = viafb_vt1636_proc_write, +static const struct proc_ops viafb_vt1636_proc_ops = { + .proc_open = viafb_vt1636_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = viafb_vt1636_proc_write, }; #endif /* CONFIG_FB_VIA_DIRECT_PROCFS */ @@ -1522,13 +1517,12 @@ static ssize_t viafb_iga1_odev_proc_write(struct file *file, return res; } -static const struct file_operations viafb_iga1_odev_proc_fops = { - .owner = THIS_MODULE, - .open = viafb_iga1_odev_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = viafb_iga1_odev_proc_write, +static const struct proc_ops viafb_iga1_odev_proc_ops = { + .proc_open = viafb_iga1_odev_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = viafb_iga1_odev_proc_write, }; static int viafb_iga2_odev_proc_show(struct seq_file *m, void *v) @@ -1562,13 +1556,12 @@ static ssize_t viafb_iga2_odev_proc_write(struct file *file, return res; } -static const struct file_operations viafb_iga2_odev_proc_fops = { - .owner = THIS_MODULE, - .open = viafb_iga2_odev_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = viafb_iga2_odev_proc_write, +static const struct proc_ops viafb_iga2_odev_proc_ops = { + .proc_open = viafb_iga2_odev_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = viafb_iga2_odev_proc_write, }; #define IS_VT1636(lvds_chip) ((lvds_chip).lvds_chip_name == VT1636_LVDS) @@ -1580,14 +1573,14 @@ static void viafb_init_proc(struct viafb_shared *shared) shared->proc_entry = viafb_entry; if (viafb_entry) { #ifdef CONFIG_FB_VIA_DIRECT_PROCFS - proc_create("dvp0", 0, viafb_entry, &viafb_dvp0_proc_fops); - proc_create("dvp1", 0, viafb_entry, &viafb_dvp1_proc_fops); - proc_create("dfph", 0, viafb_entry, &viafb_dfph_proc_fops); - proc_create("dfpl", 0, viafb_entry, &viafb_dfpl_proc_fops); + proc_create("dvp0", 0, viafb_entry, &viafb_dvp0_proc_ops); + proc_create("dvp1", 0, viafb_entry, &viafb_dvp1_proc_ops); + proc_create("dfph", 0, viafb_entry, &viafb_dfph_proc_ops); + proc_create("dfpl", 0, viafb_entry, &viafb_dfpl_proc_ops); if (IS_VT1636(shared->chip_info.lvds_chip_info) || IS_VT1636(shared->chip_info.lvds_chip_info2)) proc_create("vt1636", 0, viafb_entry, - &viafb_vt1636_proc_fops); + &viafb_vt1636_proc_ops); #endif /* CONFIG_FB_VIA_DIRECT_PROCFS */ proc_create_single("supported_output_devices", 0, viafb_entry, @@ -1595,11 +1588,11 @@ static void viafb_init_proc(struct viafb_shared *shared) iga1_entry = proc_mkdir("iga1", viafb_entry); shared->iga1_proc_entry = iga1_entry; proc_create("output_devices", 0, iga1_entry, - &viafb_iga1_odev_proc_fops); + &viafb_iga1_odev_proc_ops); iga2_entry = proc_mkdir("iga2", viafb_entry); shared->iga2_proc_entry = iga2_entry; proc_create("output_devices", 0, iga2_entry, - &viafb_iga2_odev_proc_fops); + &viafb_iga2_odev_proc_ops); } } static void viafb_remove_proc(struct viafb_shared *shared) diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 93f995f6cf36..7bfe365d9372 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -158,6 +158,8 @@ static void set_page_pfns(struct virtio_balloon *vb, { unsigned int i; + BUILD_BUG_ON(VIRTIO_BALLOON_PAGES_PER_PAGE > VIRTIO_BALLOON_ARRAY_PFNS_MAX); + /* * Set balloon pfns pointing at this page. * Note that the first pfn points at start of the page. @@ -475,7 +477,9 @@ static int init_vqs(struct virtio_balloon *vb) names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate"; callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack; names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate"; + callbacks[VIRTIO_BALLOON_VQ_STATS] = NULL; names[VIRTIO_BALLOON_VQ_STATS] = NULL; + callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { @@ -899,8 +903,7 @@ static int virtballoon_probe(struct virtio_device *vdev) vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb); if (IS_ERR(vb->vb_dev_info.inode)) { err = PTR_ERR(vb->vb_dev_info.inode); - kern_unmount(balloon_mnt); - goto out_del_vqs; + goto out_kern_unmount; } vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops; #endif @@ -911,13 +914,13 @@ static int virtballoon_probe(struct virtio_device *vdev) */ if (virtqueue_get_vring_size(vb->free_page_vq) < 2) { err = -ENOSPC; - goto out_del_vqs; + goto out_iput; } vb->balloon_wq = alloc_workqueue("balloon-wq", WQ_FREEZABLE | WQ_CPU_INTENSIVE, 0); if (!vb->balloon_wq) { err = -ENOMEM; - goto out_del_vqs; + goto out_iput; } INIT_WORK(&vb->report_free_page_work, report_free_page_func); vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP; @@ -951,6 +954,12 @@ static int virtballoon_probe(struct virtio_device *vdev) out_del_balloon_wq: if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); +out_iput: +#ifdef CONFIG_BALLOON_COMPACTION + iput(vb->vb_dev_info.inode); +out_kern_unmount: + kern_unmount(balloon_mnt); +#endif out_del_vqs: vdev->config->del_vqs(vdev); out_free_vb: @@ -966,6 +975,10 @@ static void remove_common(struct virtio_balloon *vb) leak_balloon(vb, vb->num_pages); update_balloon_size(vb); + /* There might be free pages that are being reported: release them. */ + if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + return_free_pages_to_mm(vb, ULONG_MAX); + /* Now we reset the device so we can clean up the queues. */ vb->vdev->config->reset(vb->vdev); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index e09edb5c5e06..97d5725fd9a2 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -531,18 +531,9 @@ static void virtio_mmio_release_dev(struct device *_d) static int virtio_mmio_probe(struct platform_device *pdev) { struct virtio_mmio_device *vm_dev; - struct resource *mem; unsigned long magic; int rc; - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!mem) - return -EINVAL; - - if (!devm_request_mem_region(&pdev->dev, mem->start, - resource_size(mem), pdev->name)) - return -EBUSY; - vm_dev = devm_kzalloc(&pdev->dev, sizeof(*vm_dev), GFP_KERNEL); if (!vm_dev) return -ENOMEM; @@ -554,9 +545,9 @@ static int virtio_mmio_probe(struct platform_device *pdev) INIT_LIST_HEAD(&vm_dev->virtqueues); spin_lock_init(&vm_dev->lock); - vm_dev->base = devm_ioremap(&pdev->dev, mem->start, resource_size(mem)); - if (vm_dev->base == NULL) - return -EFAULT; + vm_dev->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(vm_dev->base)) + return PTR_ERR(vm_dev->base); /* Check magic value */ magic = readl(vm_dev->base + VIRTIO_MMIO_MAGIC_VALUE); diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index f2862f66c2ac..222d630c41fc 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -294,7 +294,7 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs, /* Best option: one for change interrupt, one per vq. */ nvectors = 1; for (i = 0; i < nvqs; ++i) - if (callbacks[i]) + if (names[i] && callbacks[i]) ++nvectors; } else { /* Second best: one for change, shared for all vqs. */ diff --git a/drivers/watchdog/at91sam9_wdt.h b/drivers/watchdog/at91sam9_wdt.h index abfe34dd760a..298d545df1a1 100644 --- a/drivers/watchdog/at91sam9_wdt.h +++ b/drivers/watchdog/at91sam9_wdt.h @@ -24,7 +24,10 @@ #define AT91_WDT_MR 0x04 /* Watchdog Mode Register */ #define AT91_WDT_WDV (0xfffUL << 0) /* Counter Value */ #define AT91_WDT_SET_WDV(x) ((x) & AT91_WDT_WDV) +#define AT91_SAM9X60_PERIODRST BIT(4) /* Period Reset */ +#define AT91_SAM9X60_RPTHRST BIT(5) /* Minimum Restart Period */ #define AT91_WDT_WDFIEN BIT(12) /* Fault Interrupt Enable */ +#define AT91_SAM9X60_WDDIS BIT(12) /* Watchdog Disable */ #define AT91_WDT_WDRSTEN BIT(13) /* Reset Processor */ #define AT91_WDT_WDRPROC BIT(14) /* Timer Restart */ #define AT91_WDT_WDDIS BIT(15) /* Watchdog Disable */ @@ -37,4 +40,22 @@ #define AT91_WDT_WDUNF BIT(0) /* Watchdog Underflow */ #define AT91_WDT_WDERR BIT(1) /* Watchdog Error */ +/* Watchdog Timer Value Register */ +#define AT91_SAM9X60_VR 0x08 + +/* Watchdog Window Level Register */ +#define AT91_SAM9X60_WLR 0x0c +/* Watchdog Period Value */ +#define AT91_SAM9X60_COUNTER (0xfffUL << 0) +#define AT91_SAM9X60_SET_COUNTER(x) ((x) & AT91_SAM9X60_COUNTER) + +/* Interrupt Enable Register */ +#define AT91_SAM9X60_IER 0x14 +/* Period Interrupt Enable */ +#define AT91_SAM9X60_PERINT BIT(0) +/* Interrupt Disable Register */ +#define AT91_SAM9X60_IDR 0x18 +/* Interrupt Status Register */ +#define AT91_SAM9X60_ISR 0x1c + #endif diff --git a/drivers/watchdog/cadence_wdt.c b/drivers/watchdog/cadence_wdt.c index 06bd4e1a5923..672b184da875 100644 --- a/drivers/watchdog/cadence_wdt.c +++ b/drivers/watchdog/cadence_wdt.c @@ -369,9 +369,8 @@ static int cdns_wdt_probe(struct platform_device *pdev) return ret; platform_set_drvdata(pdev, wdt); - dev_info(dev, "Xilinx Watchdog Timer at %p with timeout %ds%s\n", - wdt->regs, cdns_wdt_device->timeout, - nowayout ? ", nowayout" : ""); + dev_info(dev, "Xilinx Watchdog Timer with timeout %ds%s\n", + cdns_wdt_device->timeout, nowayout ? ", nowayout" : ""); return 0; } diff --git a/drivers/watchdog/da9062_wdt.c b/drivers/watchdog/da9062_wdt.c index e149e66a6ea9..47eefe072b40 100644 --- a/drivers/watchdog/da9062_wdt.c +++ b/drivers/watchdog/da9062_wdt.c @@ -11,6 +11,7 @@ #include <linux/platform_device.h> #include <linux/uaccess.h> #include <linux/slab.h> +#include <linux/i2c.h> #include <linux/delay.h> #include <linux/jiffies.h> #include <linux/mfd/da9062/registers.h> @@ -147,12 +148,13 @@ static int da9062_wdt_restart(struct watchdog_device *wdd, unsigned long action, void *data) { struct da9062_watchdog *wdt = watchdog_get_drvdata(wdd); + struct i2c_client *client = to_i2c_client(wdt->hw->dev); int ret; - ret = regmap_write(wdt->hw->regmap, - DA9062AA_CONTROL_F, - DA9062AA_SHUTDOWN_MASK); - if (ret) + /* Don't use regmap because it is not atomic safe */ + ret = i2c_smbus_write_byte_data(client, DA9062AA_CONTROL_F, + DA9062AA_SHUTDOWN_MASK); + if (ret < 0) dev_alert(wdt->hw->dev, "Failed to shutdown (err = %d)\n", ret); @@ -212,6 +214,7 @@ static int da9062_wdt_probe(struct platform_device *pdev) watchdog_set_restart_priority(&wdt->wdtdev, 128); watchdog_set_drvdata(&wdt->wdtdev, wdt); + dev_set_drvdata(dev, &wdt->wdtdev); ret = devm_watchdog_register_device(dev, &wdt->wdtdev); if (ret < 0) @@ -220,10 +223,34 @@ static int da9062_wdt_probe(struct platform_device *pdev) return da9062_wdt_ping(&wdt->wdtdev); } +static int __maybe_unused da9062_wdt_suspend(struct device *dev) +{ + struct watchdog_device *wdd = dev_get_drvdata(dev); + + if (watchdog_active(wdd)) + return da9062_wdt_stop(wdd); + + return 0; +} + +static int __maybe_unused da9062_wdt_resume(struct device *dev) +{ + struct watchdog_device *wdd = dev_get_drvdata(dev); + + if (watchdog_active(wdd)) + return da9062_wdt_start(wdd); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(da9062_wdt_pm_ops, + da9062_wdt_suspend, da9062_wdt_resume); + static struct platform_driver da9062_wdt_driver = { .probe = da9062_wdt_probe, .driver = { .name = "da9062-watchdog", + .pm = &da9062_wdt_pm_ops, .of_match_table = da9062_compatible_id_table, }, }; diff --git a/drivers/watchdog/dw_wdt.c b/drivers/watchdog/dw_wdt.c index fef7c61f5555..fba21de2bbad 100644 --- a/drivers/watchdog/dw_wdt.c +++ b/drivers/watchdog/dw_wdt.c @@ -114,7 +114,15 @@ static int dw_wdt_set_timeout(struct watchdog_device *wdd, unsigned int top_s) writel(top_val | top_val << WDOG_TIMEOUT_RANGE_TOPINIT_SHIFT, dw_wdt->regs + WDOG_TIMEOUT_RANGE_REG_OFFSET); - wdd->timeout = dw_wdt_top_in_seconds(dw_wdt, top_val); + /* + * In case users set bigger timeout value than HW can support, + * kernel(watchdog_dev.c) helps to feed watchdog before + * wdd->max_hw_heartbeat_ms + */ + if (top_s * 1000 <= wdd->max_hw_heartbeat_ms) + wdd->timeout = dw_wdt_top_in_seconds(dw_wdt, top_val); + else + wdd->timeout = top_s; return 0; } @@ -135,6 +143,7 @@ static int dw_wdt_start(struct watchdog_device *wdd) struct dw_wdt *dw_wdt = to_dw_wdt(wdd); dw_wdt_set_timeout(wdd, wdd->timeout); + dw_wdt_ping(&dw_wdt->wdd); dw_wdt_arm_system_reset(dw_wdt); return 0; diff --git a/drivers/watchdog/it87_wdt.c b/drivers/watchdog/it87_wdt.c index a4b71ebc8cab..f3bf3ea50e39 100644 --- a/drivers/watchdog/it87_wdt.c +++ b/drivers/watchdog/it87_wdt.c @@ -67,6 +67,7 @@ #define IT8726_ID 0x8726 /* the data sheet suggest wrongly 0x8716 */ #define IT8728_ID 0x8728 #define IT8783_ID 0x8783 +#define IT8786_ID 0x8786 /* GPIO Configuration Registers LDN=0x07 */ #define WDTCTRL 0x71 @@ -294,6 +295,7 @@ static int __init it87_wdt_init(void) case IT8721_ID: case IT8728_ID: case IT8783_ID: + case IT8786_ID: max_units = 65535; break; case IT8705_ID: diff --git a/drivers/watchdog/mtk_wdt.c b/drivers/watchdog/mtk_wdt.c index 9c3d0033260d..d6a6393f609d 100644 --- a/drivers/watchdog/mtk_wdt.c +++ b/drivers/watchdog/mtk_wdt.c @@ -9,6 +9,9 @@ * Based on sunxi_wdt.c */ +#include <dt-bindings/reset-controller/mt2712-resets.h> +#include <dt-bindings/reset-controller/mt8183-resets.h> +#include <linux/delay.h> #include <linux/err.h> #include <linux/init.h> #include <linux/io.h> @@ -16,10 +19,11 @@ #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/reset-controller.h> #include <linux/types.h> #include <linux/watchdog.h> -#include <linux/delay.h> #define WDT_MAX_TIMEOUT 31 #define WDT_MIN_TIMEOUT 1 @@ -44,6 +48,9 @@ #define WDT_SWRST 0x14 #define WDT_SWRST_KEY 0x1209 +#define WDT_SWSYSRST 0x18U +#define WDT_SWSYS_RST_KEY 0x88000000 + #define DRV_NAME "mtk-wdt" #define DRV_VERSION "1.0" @@ -53,8 +60,94 @@ static unsigned int timeout; struct mtk_wdt_dev { struct watchdog_device wdt_dev; void __iomem *wdt_base; + spinlock_t lock; /* protects WDT_SWSYSRST reg */ + struct reset_controller_dev rcdev; +}; + +struct mtk_wdt_data { + int toprgu_sw_rst_num; }; +static const struct mtk_wdt_data mt2712_data = { + .toprgu_sw_rst_num = MT2712_TOPRGU_SW_RST_NUM, +}; + +static const struct mtk_wdt_data mt8183_data = { + .toprgu_sw_rst_num = MT8183_TOPRGU_SW_RST_NUM, +}; + +static int toprgu_reset_update(struct reset_controller_dev *rcdev, + unsigned long id, bool assert) +{ + unsigned int tmp; + unsigned long flags; + struct mtk_wdt_dev *data = + container_of(rcdev, struct mtk_wdt_dev, rcdev); + + spin_lock_irqsave(&data->lock, flags); + + tmp = readl(data->wdt_base + WDT_SWSYSRST); + if (assert) + tmp |= BIT(id); + else + tmp &= ~BIT(id); + tmp |= WDT_SWSYS_RST_KEY; + writel(tmp, data->wdt_base + WDT_SWSYSRST); + + spin_unlock_irqrestore(&data->lock, flags); + + return 0; +} + +static int toprgu_reset_assert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + return toprgu_reset_update(rcdev, id, true); +} + +static int toprgu_reset_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + return toprgu_reset_update(rcdev, id, false); +} + +static int toprgu_reset(struct reset_controller_dev *rcdev, + unsigned long id) +{ + int ret; + + ret = toprgu_reset_assert(rcdev, id); + if (ret) + return ret; + + return toprgu_reset_deassert(rcdev, id); +} + +static const struct reset_control_ops toprgu_reset_ops = { + .assert = toprgu_reset_assert, + .deassert = toprgu_reset_deassert, + .reset = toprgu_reset, +}; + +static int toprgu_register_reset_controller(struct platform_device *pdev, + int rst_num) +{ + int ret; + struct mtk_wdt_dev *mtk_wdt = platform_get_drvdata(pdev); + + spin_lock_init(&mtk_wdt->lock); + + mtk_wdt->rcdev.owner = THIS_MODULE; + mtk_wdt->rcdev.nr_resets = rst_num; + mtk_wdt->rcdev.ops = &toprgu_reset_ops; + mtk_wdt->rcdev.of_node = pdev->dev.of_node; + ret = devm_reset_controller_register(&pdev->dev, &mtk_wdt->rcdev); + if (ret != 0) + dev_err(&pdev->dev, + "couldn't register wdt reset controller: %d\n", ret); + return ret; +} + static int mtk_wdt_restart(struct watchdog_device *wdt_dev, unsigned long action, void *data) { @@ -155,6 +248,7 @@ static int mtk_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct mtk_wdt_dev *mtk_wdt; + const struct mtk_wdt_data *wdt_data; int err; mtk_wdt = devm_kzalloc(dev, sizeof(*mtk_wdt), GFP_KERNEL); @@ -190,6 +284,13 @@ static int mtk_wdt_probe(struct platform_device *pdev) dev_info(dev, "Watchdog enabled (timeout=%d sec, nowayout=%d)\n", mtk_wdt->wdt_dev.timeout, nowayout); + wdt_data = of_device_get_match_data(dev); + if (wdt_data) { + err = toprgu_register_reset_controller(pdev, + wdt_data->toprgu_sw_rst_num); + if (err) + return err; + } return 0; } @@ -218,7 +319,9 @@ static int mtk_wdt_resume(struct device *dev) #endif static const struct of_device_id mtk_wdt_dt_ids[] = { + { .compatible = "mediatek,mt2712-wdt", .data = &mt2712_data }, { .compatible = "mediatek,mt6589-wdt" }, + { .compatible = "mediatek,mt8183-wdt", .data = &mt8183_data }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, mtk_wdt_dt_ids); diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c index a494543d3ae1..eb47fe5ed280 100644 --- a/drivers/watchdog/qcom-wdt.c +++ b/drivers/watchdog/qcom-wdt.c @@ -246,7 +246,7 @@ static int qcom_wdt_probe(struct platform_device *pdev) } /* check if there is pretimeout support */ - irq = platform_get_irq(pdev, 0); + irq = platform_get_irq_optional(pdev, 0); if (irq > 0) { ret = devm_request_irq(dev, irq, qcom_wdt_isr, IRQF_TRIGGER_RISING, diff --git a/drivers/watchdog/sama5d4_wdt.c b/drivers/watchdog/sama5d4_wdt.c index d193a60430b2..e5d11d6a2600 100644 --- a/drivers/watchdog/sama5d4_wdt.c +++ b/drivers/watchdog/sama5d4_wdt.c @@ -2,7 +2,7 @@ /* * Driver for Atmel SAMA5D4 Watchdog Timer * - * Copyright (C) 2015 Atmel Corporation + * Copyright (C) 2015-2019 Microchip Technology Inc. and its subsidiaries */ #include <linux/delay.h> @@ -11,6 +11,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/platform_device.h> #include <linux/reboot.h> @@ -29,7 +30,10 @@ struct sama5d4_wdt { struct watchdog_device wdd; void __iomem *reg_base; u32 mr; + u32 ir; unsigned long last_ping; + bool need_irq; + bool sam9x60_support; }; static int wdt_timeout; @@ -78,7 +82,12 @@ static int sama5d4_wdt_start(struct watchdog_device *wdd) { struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd); - wdt->mr &= ~AT91_WDT_WDDIS; + if (wdt->sam9x60_support) { + writel_relaxed(wdt->ir, wdt->reg_base + AT91_SAM9X60_IER); + wdt->mr &= ~AT91_SAM9X60_WDDIS; + } else { + wdt->mr &= ~AT91_WDT_WDDIS; + } wdt_write(wdt, AT91_WDT_MR, wdt->mr); return 0; @@ -88,7 +97,12 @@ static int sama5d4_wdt_stop(struct watchdog_device *wdd) { struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd); - wdt->mr |= AT91_WDT_WDDIS; + if (wdt->sam9x60_support) { + writel_relaxed(wdt->ir, wdt->reg_base + AT91_SAM9X60_IDR); + wdt->mr |= AT91_SAM9X60_WDDIS; + } else { + wdt->mr |= AT91_WDT_WDDIS; + } wdt_write(wdt, AT91_WDT_MR, wdt->mr); return 0; @@ -109,6 +123,14 @@ static int sama5d4_wdt_set_timeout(struct watchdog_device *wdd, struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd); u32 value = WDT_SEC2TICKS(timeout); + if (wdt->sam9x60_support) { + wdt_write(wdt, AT91_SAM9X60_WLR, + AT91_SAM9X60_SET_COUNTER(value)); + + wdd->timeout = timeout; + return 0; + } + wdt->mr &= ~AT91_WDT_WDV; wdt->mr |= AT91_WDT_SET_WDV(value); @@ -143,8 +165,14 @@ static const struct watchdog_ops sama5d4_wdt_ops = { static irqreturn_t sama5d4_wdt_irq_handler(int irq, void *dev_id) { struct sama5d4_wdt *wdt = platform_get_drvdata(dev_id); + u32 reg; - if (wdt_read(wdt, AT91_WDT_SR)) { + if (wdt->sam9x60_support) + reg = wdt_read(wdt, AT91_SAM9X60_ISR); + else + reg = wdt_read(wdt, AT91_WDT_SR); + + if (reg) { pr_crit("Atmel Watchdog Software Reset\n"); emergency_restart(); pr_crit("Reboot didn't succeed\n"); @@ -157,13 +185,14 @@ static int of_sama5d4_wdt_init(struct device_node *np, struct sama5d4_wdt *wdt) { const char *tmp; - wdt->mr = AT91_WDT_WDDIS; + if (wdt->sam9x60_support) + wdt->mr = AT91_SAM9X60_WDDIS; + else + wdt->mr = AT91_WDT_WDDIS; if (!of_property_read_string(np, "atmel,watchdog-type", &tmp) && !strcmp(tmp, "software")) - wdt->mr |= AT91_WDT_WDFIEN; - else - wdt->mr |= AT91_WDT_WDRSTEN; + wdt->need_irq = true; if (of_property_read_bool(np, "atmel,idle-halt")) wdt->mr |= AT91_WDT_WDIDLEHLT; @@ -176,21 +205,46 @@ static int of_sama5d4_wdt_init(struct device_node *np, struct sama5d4_wdt *wdt) static int sama5d4_wdt_init(struct sama5d4_wdt *wdt) { - u32 reg; + u32 reg, val; + + val = WDT_SEC2TICKS(WDT_DEFAULT_TIMEOUT); /* * When booting and resuming, the bootloader may have changed the * watchdog configuration. * If the watchdog is already running, we can safely update it. * Else, we have to disable it properly. */ - if (wdt_enabled) { - wdt_write_nosleep(wdt, AT91_WDT_MR, wdt->mr); - } else { + if (!wdt_enabled) { reg = wdt_read(wdt, AT91_WDT_MR); - if (!(reg & AT91_WDT_WDDIS)) + if (wdt->sam9x60_support && (!(reg & AT91_SAM9X60_WDDIS))) + wdt_write_nosleep(wdt, AT91_WDT_MR, + reg | AT91_SAM9X60_WDDIS); + else if (!wdt->sam9x60_support && + (!(reg & AT91_WDT_WDDIS))) wdt_write_nosleep(wdt, AT91_WDT_MR, reg | AT91_WDT_WDDIS); } + + if (wdt->sam9x60_support) { + if (wdt->need_irq) + wdt->ir = AT91_SAM9X60_PERINT; + else + wdt->mr |= AT91_SAM9X60_PERIODRST; + + wdt_write(wdt, AT91_SAM9X60_IER, wdt->ir); + wdt_write(wdt, AT91_SAM9X60_WLR, AT91_SAM9X60_SET_COUNTER(val)); + } else { + wdt->mr |= AT91_WDT_SET_WDD(WDT_SEC2TICKS(MAX_WDT_TIMEOUT)); + wdt->mr |= AT91_WDT_SET_WDV(val); + + if (wdt->need_irq) + wdt->mr |= AT91_WDT_WDFIEN; + else + wdt->mr |= AT91_WDT_WDRSTEN; + } + + wdt_write_nosleep(wdt, AT91_WDT_MR, wdt->mr); + return 0; } @@ -201,7 +255,6 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) struct sama5d4_wdt *wdt; void __iomem *regs; u32 irq = 0; - u32 timeout; int ret; wdt = devm_kzalloc(dev, sizeof(*wdt), GFP_KERNEL); @@ -215,6 +268,8 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) wdd->min_timeout = MIN_WDT_TIMEOUT; wdd->max_timeout = MAX_WDT_TIMEOUT; wdt->last_ping = jiffies; + wdt->sam9x60_support = of_device_is_compatible(dev->of_node, + "microchip,sam9x60-wdt"); watchdog_set_drvdata(wdd, wdt); @@ -224,15 +279,19 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) wdt->reg_base = regs; - irq = irq_of_parse_and_map(dev->of_node, 0); - if (!irq) - dev_warn(dev, "failed to get IRQ from DT\n"); - ret = of_sama5d4_wdt_init(dev->of_node, wdt); if (ret) return ret; - if ((wdt->mr & AT91_WDT_WDFIEN) && irq) { + if (wdt->need_irq) { + irq = irq_of_parse_and_map(dev->of_node, 0); + if (!irq) { + dev_warn(dev, "failed to get IRQ from DT\n"); + wdt->need_irq = false; + } + } + + if (wdt->need_irq) { ret = devm_request_irq(dev, irq, sama5d4_wdt_irq_handler, IRQF_SHARED | IRQF_IRQPOLL | IRQF_NO_SUSPEND, pdev->name, pdev); @@ -244,11 +303,6 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) watchdog_init_timeout(wdd, wdt_timeout, dev); - timeout = WDT_SEC2TICKS(wdd->timeout); - - wdt->mr |= AT91_WDT_SET_WDD(WDT_SEC2TICKS(MAX_WDT_TIMEOUT)); - wdt->mr |= AT91_WDT_SET_WDV(timeout); - ret = sama5d4_wdt_init(wdt); if (ret) return ret; @@ -269,7 +323,12 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) } static const struct of_device_id sama5d4_wdt_of_match[] = { - { .compatible = "atmel,sama5d4-wdt", }, + { + .compatible = "atmel,sama5d4-wdt", + }, + { + .compatible = "microchip,sam9x60-wdt", + }, { } }; MODULE_DEVICE_TABLE(of, sama5d4_wdt_of_match); diff --git a/drivers/watchdog/stm32_iwdg.c b/drivers/watchdog/stm32_iwdg.c index a3a329011a06..25188d6bbe15 100644 --- a/drivers/watchdog/stm32_iwdg.c +++ b/drivers/watchdog/stm32_iwdg.c @@ -262,6 +262,24 @@ static int stm32_iwdg_probe(struct platform_device *pdev) watchdog_set_nowayout(wdd, WATCHDOG_NOWAYOUT); watchdog_init_timeout(wdd, 0, dev); + /* + * In case of CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED is set + * (Means U-Boot/bootloaders leaves the watchdog running) + * When we get here we should make a decision to prevent + * any side effects before user space daemon will take care of it. + * The best option, taking into consideration that there is no + * way to read values back from hardware, is to enforce watchdog + * being run with deterministic values. + */ + if (IS_ENABLED(CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED)) { + ret = stm32_iwdg_start(wdd); + if (ret) + return ret; + + /* Make sure the watchdog is serviced */ + set_bit(WDOG_HW_RUNNING, &wdd->status); + } + ret = devm_watchdog_register_device(dev, wdd); if (ret) return ret; diff --git a/drivers/watchdog/watchdog_core.c b/drivers/watchdog/watchdog_core.c index 21e8085b848b..861daf4f37b2 100644 --- a/drivers/watchdog/watchdog_core.c +++ b/drivers/watchdog/watchdog_core.c @@ -147,6 +147,25 @@ int watchdog_init_timeout(struct watchdog_device *wdd, } EXPORT_SYMBOL_GPL(watchdog_init_timeout); +static int watchdog_reboot_notifier(struct notifier_block *nb, + unsigned long code, void *data) +{ + struct watchdog_device *wdd; + + wdd = container_of(nb, struct watchdog_device, reboot_nb); + if (code == SYS_DOWN || code == SYS_HALT) { + if (watchdog_active(wdd)) { + int ret; + + ret = wdd->ops->stop(wdd); + if (ret) + return NOTIFY_BAD; + } + } + + return NOTIFY_DONE; +} + static int watchdog_restart_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -235,6 +254,19 @@ static int __watchdog_register_device(struct watchdog_device *wdd) } } + if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status)) { + wdd->reboot_nb.notifier_call = watchdog_reboot_notifier; + + ret = register_reboot_notifier(&wdd->reboot_nb); + if (ret) { + pr_err("watchdog%d: Cannot register reboot notifier (%d)\n", + wdd->id, ret); + watchdog_dev_unregister(wdd); + ida_simple_remove(&watchdog_ida, id); + return ret; + } + } + if (wdd->ops->restart) { wdd->restart_nb.notifier_call = watchdog_restart_notifier; @@ -289,6 +321,9 @@ static void __watchdog_unregister_device(struct watchdog_device *wdd) if (wdd->ops->restart) unregister_restart_handler(&wdd->restart_nb); + if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status)) + unregister_reboot_notifier(&wdd->reboot_nb); + watchdog_dev_unregister(wdd); ida_simple_remove(&watchdog_ida, wdd->id); } diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c index 4b2a85438478..8b5c742f24e8 100644 --- a/drivers/watchdog/watchdog_dev.c +++ b/drivers/watchdog/watchdog_dev.c @@ -38,7 +38,6 @@ #include <linux/miscdevice.h> /* For handling misc devices */ #include <linux/module.h> /* For module stuff/... */ #include <linux/mutex.h> /* For mutexes */ -#include <linux/reboot.h> /* For reboot notifier */ #include <linux/slab.h> /* For memory functions */ #include <linux/types.h> /* For standard types (like size_t) */ #include <linux/watchdog.h> /* For watchdog specific items */ @@ -1097,25 +1096,6 @@ static void watchdog_cdev_unregister(struct watchdog_device *wdd) put_device(&wd_data->dev); } -static int watchdog_reboot_notifier(struct notifier_block *nb, - unsigned long code, void *data) -{ - struct watchdog_device *wdd; - - wdd = container_of(nb, struct watchdog_device, reboot_nb); - if (code == SYS_DOWN || code == SYS_HALT) { - if (watchdog_active(wdd)) { - int ret; - - ret = wdd->ops->stop(wdd); - if (ret) - return NOTIFY_BAD; - } - } - - return NOTIFY_DONE; -} - /* * watchdog_dev_register: register a watchdog device * @wdd: watchdog device @@ -1134,22 +1114,8 @@ int watchdog_dev_register(struct watchdog_device *wdd) return ret; ret = watchdog_register_pretimeout(wdd); - if (ret) { + if (ret) watchdog_cdev_unregister(wdd); - return ret; - } - - if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status)) { - wdd->reboot_nb.notifier_call = watchdog_reboot_notifier; - - ret = devm_register_reboot_notifier(&wdd->wd_data->dev, - &wdd->reboot_nb); - if (ret) { - pr_err("watchdog%d: Cannot register reboot notifier (%d)\n", - wdd->id, ret); - watchdog_dev_unregister(wdd); - } - } return ret; } diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 4fc83e3f5ad3..0258415ca0b2 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -1006,19 +1006,19 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) } mutex_unlock(&priv->lock); - /* - * gntdev takes the address of the PTE in find_grant_ptes() and passes - * it to the hypervisor in gntdev_map_grant_pages(). The purpose of - * the notifier is to prevent the hypervisor pointer to the PTE from - * going stale. - * - * Since this vma's mappings can't be touched without the mmap_sem, - * and we are holding it now, there is no need for the notifier_range - * locking pattern. - */ - mmu_interval_read_begin(&map->notifier); - if (use_ptemod) { + /* + * gntdev takes the address of the PTE in find_grant_ptes() and + * passes it to the hypervisor in gntdev_map_grant_pages(). The + * purpose of the notifier is to prevent the hypervisor pointer + * to the PTE from going stale. + * + * Since this vma's mappings can't be touched without the + * mmap_sem, and we are holding it now, there is no need for + * the notifier_range locking pattern. + */ + mmu_interval_read_begin(&map->notifier); + map->pages_vm_start = vma->vm_start; err = apply_to_page_range(vma->vm_mm, vma->vm_start, vma->vm_end - vma->vm_start, diff --git a/drivers/xen/xen-balloon.c b/drivers/xen/xen-balloon.c index 6d12fc368210..a8d24433c8e9 100644 --- a/drivers/xen/xen-balloon.c +++ b/drivers/xen/xen-balloon.c @@ -94,7 +94,7 @@ static void watch_target(struct xenbus_watch *watch, "%llu", &static_max) == 1)) static_max >>= PAGE_SHIFT - 10; else - static_max = new_target; + static_max = balloon_stats.current_pages; target_diff = (xen_pv_domain() || xen_initial_domain()) ? 0 : static_max - balloon_stats.target_pages; diff --git a/drivers/xen/xen-pciback/conf_space.c b/drivers/xen/xen-pciback/conf_space.c index 60111719b01f..b20e43e148ce 100644 --- a/drivers/xen/xen-pciback/conf_space.c +++ b/drivers/xen/xen-pciback/conf_space.c @@ -286,6 +286,43 @@ int xen_pcibk_config_write(struct pci_dev *dev, int offset, int size, u32 value) return xen_pcibios_err_to_errno(err); } +int xen_pcibk_get_interrupt_type(struct pci_dev *dev) +{ + int err; + u16 val; + int ret = 0; + + err = pci_read_config_word(dev, PCI_COMMAND, &val); + if (err) + return err; + if (!(val & PCI_COMMAND_INTX_DISABLE)) + ret |= INTERRUPT_TYPE_INTX; + + /* + * Do not trust dev->msi(x)_enabled here, as enabling could be done + * bypassing the pci_*msi* functions, by the qemu. + */ + if (dev->msi_cap) { + err = pci_read_config_word(dev, + dev->msi_cap + PCI_MSI_FLAGS, + &val); + if (err) + return err; + if (val & PCI_MSI_FLAGS_ENABLE) + ret |= INTERRUPT_TYPE_MSI; + } + if (dev->msix_cap) { + err = pci_read_config_word(dev, + dev->msix_cap + PCI_MSIX_FLAGS, + &val); + if (err) + return err; + if (val & PCI_MSIX_FLAGS_ENABLE) + ret |= INTERRUPT_TYPE_MSIX; + } + return ret; +} + void xen_pcibk_config_free_dyn_fields(struct pci_dev *dev) { struct xen_pcibk_dev_data *dev_data = pci_get_drvdata(dev); diff --git a/drivers/xen/xen-pciback/conf_space.h b/drivers/xen/xen-pciback/conf_space.h index 22db630717ea..28c45180a12e 100644 --- a/drivers/xen/xen-pciback/conf_space.h +++ b/drivers/xen/xen-pciback/conf_space.h @@ -65,6 +65,11 @@ struct config_field_entry { void *data; }; +#define INTERRUPT_TYPE_NONE (1<<0) +#define INTERRUPT_TYPE_INTX (1<<1) +#define INTERRUPT_TYPE_MSI (1<<2) +#define INTERRUPT_TYPE_MSIX (1<<3) + extern bool xen_pcibk_permissive; #define OFFSET(cfg_entry) ((cfg_entry)->base_offset+(cfg_entry)->field->offset) @@ -126,4 +131,6 @@ int xen_pcibk_config_capability_init(void); int xen_pcibk_config_header_add_fields(struct pci_dev *dev); int xen_pcibk_config_capability_add_fields(struct pci_dev *dev); +int xen_pcibk_get_interrupt_type(struct pci_dev *dev); + #endif /* __XEN_PCIBACK_CONF_SPACE_H__ */ diff --git a/drivers/xen/xen-pciback/conf_space_capability.c b/drivers/xen/xen-pciback/conf_space_capability.c index e5694133ebe5..22f13abbe913 100644 --- a/drivers/xen/xen-pciback/conf_space_capability.c +++ b/drivers/xen/xen-pciback/conf_space_capability.c @@ -189,6 +189,85 @@ static const struct config_field caplist_pm[] = { {} }; +static struct msi_msix_field_config { + u16 enable_bit; /* bit for enabling MSI/MSI-X */ + unsigned int int_type; /* interrupt type for exclusiveness check */ +} msi_field_config = { + .enable_bit = PCI_MSI_FLAGS_ENABLE, + .int_type = INTERRUPT_TYPE_MSI, +}, msix_field_config = { + .enable_bit = PCI_MSIX_FLAGS_ENABLE, + .int_type = INTERRUPT_TYPE_MSIX, +}; + +static void *msi_field_init(struct pci_dev *dev, int offset) +{ + return &msi_field_config; +} + +static void *msix_field_init(struct pci_dev *dev, int offset) +{ + return &msix_field_config; +} + +static int msi_msix_flags_write(struct pci_dev *dev, int offset, u16 new_value, + void *data) +{ + int err; + u16 old_value; + const struct msi_msix_field_config *field_config = data; + const struct xen_pcibk_dev_data *dev_data = pci_get_drvdata(dev); + + if (xen_pcibk_permissive || dev_data->permissive) + goto write; + + err = pci_read_config_word(dev, offset, &old_value); + if (err) + return err; + + if (new_value == old_value) + return 0; + + if (!dev_data->allow_interrupt_control || + (new_value ^ old_value) & ~field_config->enable_bit) + return PCIBIOS_SET_FAILED; + + if (new_value & field_config->enable_bit) { + /* don't allow enabling together with other interrupt types */ + int int_type = xen_pcibk_get_interrupt_type(dev); + + if (int_type == INTERRUPT_TYPE_NONE || + int_type == field_config->int_type) + goto write; + return PCIBIOS_SET_FAILED; + } + +write: + return pci_write_config_word(dev, offset, new_value); +} + +static const struct config_field caplist_msix[] = { + { + .offset = PCI_MSIX_FLAGS, + .size = 2, + .init = msix_field_init, + .u.w.read = xen_pcibk_read_config_word, + .u.w.write = msi_msix_flags_write, + }, + {} +}; + +static const struct config_field caplist_msi[] = { + { + .offset = PCI_MSI_FLAGS, + .size = 2, + .init = msi_field_init, + .u.w.read = xen_pcibk_read_config_word, + .u.w.write = msi_msix_flags_write, + }, + {} +}; + static struct xen_pcibk_config_capability xen_pcibk_config_capability_pm = { .capability = PCI_CAP_ID_PM, .fields = caplist_pm, @@ -197,11 +276,21 @@ static struct xen_pcibk_config_capability xen_pcibk_config_capability_vpd = { .capability = PCI_CAP_ID_VPD, .fields = caplist_vpd, }; +static struct xen_pcibk_config_capability xen_pcibk_config_capability_msi = { + .capability = PCI_CAP_ID_MSI, + .fields = caplist_msi, +}; +static struct xen_pcibk_config_capability xen_pcibk_config_capability_msix = { + .capability = PCI_CAP_ID_MSIX, + .fields = caplist_msix, +}; int xen_pcibk_config_capability_init(void) { register_capability(&xen_pcibk_config_capability_vpd); register_capability(&xen_pcibk_config_capability_pm); + register_capability(&xen_pcibk_config_capability_msi); + register_capability(&xen_pcibk_config_capability_msix); return 0; } diff --git a/drivers/xen/xen-pciback/conf_space_header.c b/drivers/xen/xen-pciback/conf_space_header.c index 10ae24b5a76e..fb4fccb4aecc 100644 --- a/drivers/xen/xen-pciback/conf_space_header.c +++ b/drivers/xen/xen-pciback/conf_space_header.c @@ -117,6 +117,25 @@ static int command_write(struct pci_dev *dev, int offset, u16 value, void *data) pci_clear_mwi(dev); } + if (dev_data && dev_data->allow_interrupt_control) { + if ((cmd->val ^ value) & PCI_COMMAND_INTX_DISABLE) { + if (value & PCI_COMMAND_INTX_DISABLE) { + pci_intx(dev, 0); + } else { + /* Do not allow enabling INTx together with MSI or MSI-X. */ + switch (xen_pcibk_get_interrupt_type(dev)) { + case INTERRUPT_TYPE_NONE: + pci_intx(dev, 1); + break; + case INTERRUPT_TYPE_INTX: + break; + default: + return PCIBIOS_SET_FAILED; + } + } + } + } + cmd->val = value; if (!xen_pcibk_permissive && (!dev_data || !dev_data->permissive)) diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index 097410a7cdb7..7af93d65ed51 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -304,6 +304,8 @@ void pcistub_put_pci_dev(struct pci_dev *dev) xen_pcibk_config_reset_dev(dev); xen_pcibk_config_free_dyn_fields(dev); + dev_data->allow_interrupt_control = 0; + xen_unregister_device_domain_owner(dev); spin_lock_irqsave(&found_psdev->lock, flags); @@ -1431,6 +1433,65 @@ static ssize_t permissive_show(struct device_driver *drv, char *buf) } static DRIVER_ATTR_RW(permissive); +static ssize_t allow_interrupt_control_store(struct device_driver *drv, + const char *buf, size_t count) +{ + int domain, bus, slot, func; + int err; + struct pcistub_device *psdev; + struct xen_pcibk_dev_data *dev_data; + + err = str_to_slot(buf, &domain, &bus, &slot, &func); + if (err) + goto out; + + psdev = pcistub_device_find(domain, bus, slot, func); + if (!psdev) { + err = -ENODEV; + goto out; + } + + dev_data = pci_get_drvdata(psdev->dev); + /* the driver data for a device should never be null at this point */ + if (!dev_data) { + err = -ENXIO; + goto release; + } + dev_data->allow_interrupt_control = 1; +release: + pcistub_device_put(psdev); +out: + if (!err) + err = count; + return err; +} + +static ssize_t allow_interrupt_control_show(struct device_driver *drv, + char *buf) +{ + struct pcistub_device *psdev; + struct xen_pcibk_dev_data *dev_data; + size_t count = 0; + unsigned long flags; + + spin_lock_irqsave(&pcistub_devices_lock, flags); + list_for_each_entry(psdev, &pcistub_devices, dev_list) { + if (count >= PAGE_SIZE) + break; + if (!psdev->dev) + continue; + dev_data = pci_get_drvdata(psdev->dev); + if (!dev_data || !dev_data->allow_interrupt_control) + continue; + count += + scnprintf(buf + count, PAGE_SIZE - count, "%s\n", + pci_name(psdev->dev)); + } + spin_unlock_irqrestore(&pcistub_devices_lock, flags); + return count; +} +static DRIVER_ATTR_RW(allow_interrupt_control); + static void pcistub_exit(void) { driver_remove_file(&xen_pcibk_pci_driver.driver, &driver_attr_new_slot); @@ -1441,6 +1502,8 @@ static void pcistub_exit(void) driver_remove_file(&xen_pcibk_pci_driver.driver, &driver_attr_permissive); driver_remove_file(&xen_pcibk_pci_driver.driver, + &driver_attr_allow_interrupt_control); + driver_remove_file(&xen_pcibk_pci_driver.driver, &driver_attr_irq_handlers); driver_remove_file(&xen_pcibk_pci_driver.driver, &driver_attr_irq_handler_state); @@ -1530,6 +1593,9 @@ static int __init pcistub_init(void) if (!err) err = driver_create_file(&xen_pcibk_pci_driver.driver, &driver_attr_permissive); + if (!err) + err = driver_create_file(&xen_pcibk_pci_driver.driver, + &driver_attr_allow_interrupt_control); if (!err) err = driver_create_file(&xen_pcibk_pci_driver.driver, diff --git a/drivers/xen/xen-pciback/pciback.h b/drivers/xen/xen-pciback/pciback.h index 263c059bff90..ce1077e32466 100644 --- a/drivers/xen/xen-pciback/pciback.h +++ b/drivers/xen/xen-pciback/pciback.h @@ -45,6 +45,7 @@ struct xen_pcibk_dev_data { struct list_head config_fields; struct pci_saved_state *pci_saved_state; unsigned int permissive:1; + unsigned int allow_interrupt_control:1; unsigned int warned_on_write:1; unsigned int enable_intx:1; unsigned int isr_on:1; /* Whether the IRQ handler is installed. */ diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index 378486b79f96..66975da4f3b6 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -239,7 +239,9 @@ int xenbus_dev_probe(struct device *_dev) goto fail; } + spin_lock(&dev->reclaim_lock); err = drv->probe(dev, id); + spin_unlock(&dev->reclaim_lock); if (err) goto fail_put; @@ -268,8 +270,11 @@ int xenbus_dev_remove(struct device *_dev) free_otherend_watch(dev); - if (drv->remove) + if (drv->remove) { + spin_lock(&dev->reclaim_lock); drv->remove(dev); + spin_unlock(&dev->reclaim_lock); + } module_put(drv->driver.owner); @@ -468,6 +473,7 @@ int xenbus_probe_node(struct xen_bus_type *bus, goto fail; dev_set_name(&xendev->dev, "%s", devname); + spin_lock_init(&xendev->reclaim_lock); /* Register with generic device framework. */ err = device_register(&xendev->dev); diff --git a/drivers/xen/xenbus/xenbus_probe_backend.c b/drivers/xen/xenbus/xenbus_probe_backend.c index 14876faff3b0..791f6fe01e91 100644 --- a/drivers/xen/xenbus/xenbus_probe_backend.c +++ b/drivers/xen/xenbus/xenbus_probe_backend.c @@ -247,6 +247,41 @@ static int backend_probe_and_watch(struct notifier_block *notifier, return NOTIFY_DONE; } +static int backend_reclaim_memory(struct device *dev, void *data) +{ + const struct xenbus_driver *drv; + struct xenbus_device *xdev; + + if (!dev->driver) + return 0; + drv = to_xenbus_driver(dev->driver); + if (drv && drv->reclaim_memory) { + xdev = to_xenbus_device(dev); + if (!spin_trylock(&xdev->reclaim_lock)) + return 0; + drv->reclaim_memory(xdev); + spin_unlock(&xdev->reclaim_lock); + } + return 0; +} + +/* + * Returns 0 always because we are using shrinker to only detect memory + * pressure. + */ +static unsigned long backend_shrink_memory_count(struct shrinker *shrinker, + struct shrink_control *sc) +{ + bus_for_each_dev(&xenbus_backend.bus, NULL, NULL, + backend_reclaim_memory); + return 0; +} + +static struct shrinker backend_memory_shrinker = { + .count_objects = backend_shrink_memory_count, + .seeks = DEFAULT_SEEKS, +}; + static int __init xenbus_probe_backend_init(void) { static struct notifier_block xenstore_notifier = { @@ -263,6 +298,9 @@ static int __init xenbus_probe_backend_init(void) register_xenstore_notifier(&xenstore_notifier); + if (register_shrinker(&backend_memory_shrinker)) + pr_warn("shrinker registration failed\n"); + return 0; } subsys_initcall(xenbus_probe_backend_init); diff --git a/drivers/zorro/proc.c b/drivers/zorro/proc.c index 2e4ca4dc0960..1c9ae08225d8 100644 --- a/drivers/zorro/proc.c +++ b/drivers/zorro/proc.c @@ -56,10 +56,9 @@ proc_bus_zorro_read(struct file *file, char __user *buf, size_t nbytes, loff_t * return nbytes; } -static const struct file_operations proc_bus_zorro_operations = { - .owner = THIS_MODULE, - .llseek = proc_bus_zorro_lseek, - .read = proc_bus_zorro_read, +static const struct proc_ops bus_zorro_proc_ops = { + .proc_lseek = proc_bus_zorro_lseek, + .proc_read = proc_bus_zorro_read, }; static void * zorro_seq_start(struct seq_file *m, loff_t *pos) @@ -105,7 +104,7 @@ static int __init zorro_proc_attach_device(unsigned int slot) sprintf(name, "%02x", slot); entry = proc_create_data(name, 0, proc_bus_zorro_dir, - &proc_bus_zorro_operations, + &bus_zorro_proc_ops, &zorro_autocon[slot]); if (!entry) return -ENOMEM; |