diff options
Diffstat (limited to 'drivers/hwtracing')
18 files changed, 1179 insertions, 488 deletions
diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index ff94e58845b7..170fbb66bda2 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -406,6 +406,7 @@ static inline int catu_wait_for_ready(struct catu_drvdata *drvdata) static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) { + int rc; u32 control, mode; struct etr_buf *etr_buf = data; @@ -418,6 +419,10 @@ static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) return -EBUSY; } + rc = coresight_claim_device_unlocked(drvdata->base); + if (rc) + return rc; + control |= BIT(CATU_CONTROL_ENABLE); if (etr_buf && etr_buf->mode == ETR_MODE_CATU) { @@ -459,6 +464,7 @@ static int catu_disable_hw(struct catu_drvdata *drvdata) int rc = 0; catu_write_control(drvdata, 0); + coresight_disclaim_device_unlocked(drvdata->base); if (catu_wait_for_ready(drvdata)) { dev_info(drvdata->dev, "Timeout while waiting for READY\n"); rc = -EAGAIN; diff --git a/drivers/hwtracing/coresight/coresight-dynamic-replicator.c b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c index f6d0571ab9dd..299667b887fc 100644 --- a/drivers/hwtracing/coresight/coresight-dynamic-replicator.c +++ b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c @@ -34,48 +34,87 @@ struct replicator_state { struct coresight_device *csdev; }; +/* + * replicator_reset : Reset the replicator configuration to sane values. + */ +static void replicator_reset(struct replicator_state *drvdata) +{ + CS_UNLOCK(drvdata->base); + + if (!coresight_claim_device_unlocked(drvdata->base)) { + writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER0); + writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER1); + coresight_disclaim_device_unlocked(drvdata->base); + } + + CS_LOCK(drvdata->base); +} + static int replicator_enable(struct coresight_device *csdev, int inport, int outport) { + int rc = 0; + u32 reg; struct replicator_state *drvdata = dev_get_drvdata(csdev->dev.parent); + switch (outport) { + case 0: + reg = REPLICATOR_IDFILTER0; + break; + case 1: + reg = REPLICATOR_IDFILTER1; + break; + default: + WARN_ON(1); + return -EINVAL; + } + CS_UNLOCK(drvdata->base); - /* - * Ensure that the other port is disabled - * 0x00 - passing through the replicator unimpeded - * 0xff - disable (or impede) the flow of ATB data - */ - if (outport == 0) { - writel_relaxed(0x00, drvdata->base + REPLICATOR_IDFILTER0); - writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER1); - } else { - writel_relaxed(0x00, drvdata->base + REPLICATOR_IDFILTER1); - writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER0); + if ((readl_relaxed(drvdata->base + REPLICATOR_IDFILTER0) == 0xff) && + (readl_relaxed(drvdata->base + REPLICATOR_IDFILTER1) == 0xff)) + rc = coresight_claim_device_unlocked(drvdata->base); + + /* Ensure that the outport is enabled. */ + if (!rc) { + writel_relaxed(0x00, drvdata->base + reg); + dev_dbg(drvdata->dev, "REPLICATOR enabled\n"); } CS_LOCK(drvdata->base); - dev_info(drvdata->dev, "REPLICATOR enabled\n"); - return 0; + return rc; } static void replicator_disable(struct coresight_device *csdev, int inport, int outport) { + u32 reg; struct replicator_state *drvdata = dev_get_drvdata(csdev->dev.parent); + switch (outport) { + case 0: + reg = REPLICATOR_IDFILTER0; + break; + case 1: + reg = REPLICATOR_IDFILTER1; + break; + default: + WARN_ON(1); + return; + } + CS_UNLOCK(drvdata->base); /* disable the flow of ATB data through port */ - if (outport == 0) - writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER0); - else - writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER1); + writel_relaxed(0xff, drvdata->base + reg); + if ((readl_relaxed(drvdata->base + REPLICATOR_IDFILTER0) == 0xff) && + (readl_relaxed(drvdata->base + REPLICATOR_IDFILTER1) == 0xff)) + coresight_disclaim_device_unlocked(drvdata->base); CS_LOCK(drvdata->base); - dev_info(drvdata->dev, "REPLICATOR disabled\n"); + dev_dbg(drvdata->dev, "REPLICATOR disabled\n"); } static const struct coresight_ops_link replicator_link_ops = { @@ -156,7 +195,11 @@ static int replicator_probe(struct amba_device *adev, const struct amba_id *id) desc.groups = replicator_groups; drvdata->csdev = coresight_register(&desc); - return PTR_ERR_OR_ZERO(drvdata->csdev); + if (!IS_ERR(drvdata->csdev)) { + replicator_reset(drvdata); + return 0; + } + return PTR_ERR(drvdata->csdev); } #ifdef CONFIG_PM diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 306119eaf16a..824be0c5f592 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -5,7 +5,6 @@ * Description: CoreSight Embedded Trace Buffer driver */ -#include <asm/local.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/types.h> @@ -28,6 +27,7 @@ #include "coresight-priv.h" +#include "coresight-etm-perf.h" #define ETB_RAM_DEPTH_REG 0x004 #define ETB_STATUS_REG 0x00c @@ -71,8 +71,8 @@ * @miscdev: specifics to handle "/dev/xyz.etb" entry. * @spinlock: only one at a time pls. * @reading: synchronise user space access to etb buffer. - * @mode: this ETB is being used. * @buf: area of memory where ETB buffer content gets sent. + * @mode: this ETB is being used. * @buffer_depth: size of @buf. * @trigger_cntr: amount of words to store after a trigger. */ @@ -84,12 +84,15 @@ struct etb_drvdata { struct miscdevice miscdev; spinlock_t spinlock; local_t reading; - local_t mode; u8 *buf; + u32 mode; u32 buffer_depth; u32 trigger_cntr; }; +static int etb_set_buffer(struct coresight_device *csdev, + struct perf_output_handle *handle); + static unsigned int etb_get_buffer_depth(struct etb_drvdata *drvdata) { u32 depth = 0; @@ -103,7 +106,7 @@ static unsigned int etb_get_buffer_depth(struct etb_drvdata *drvdata) return depth; } -static void etb_enable_hw(struct etb_drvdata *drvdata) +static void __etb_enable_hw(struct etb_drvdata *drvdata) { int i; u32 depth; @@ -131,32 +134,92 @@ static void etb_enable_hw(struct etb_drvdata *drvdata) CS_LOCK(drvdata->base); } -static int etb_enable(struct coresight_device *csdev, u32 mode) +static int etb_enable_hw(struct etb_drvdata *drvdata) +{ + __etb_enable_hw(drvdata); + return 0; +} + +static int etb_enable_sysfs(struct coresight_device *csdev) { - u32 val; + int ret = 0; unsigned long flags; struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - val = local_cmpxchg(&drvdata->mode, - CS_MODE_DISABLED, mode); - /* - * When accessing from Perf, a HW buffer can be handled - * by a single trace entity. In sysFS mode many tracers - * can be logging to the same HW buffer. - */ - if (val == CS_MODE_PERF) - return -EBUSY; + spin_lock_irqsave(&drvdata->spinlock, flags); + + /* Don't messup with perf sessions. */ + if (drvdata->mode == CS_MODE_PERF) { + ret = -EBUSY; + goto out; + } /* Nothing to do, the tracer is already enabled. */ - if (val == CS_MODE_SYSFS) + if (drvdata->mode == CS_MODE_SYSFS) goto out; - spin_lock_irqsave(&drvdata->spinlock, flags); - etb_enable_hw(drvdata); + ret = etb_enable_hw(drvdata); + if (!ret) + drvdata->mode = CS_MODE_SYSFS; + +out: spin_unlock_irqrestore(&drvdata->spinlock, flags); + return ret; +} + +static int etb_enable_perf(struct coresight_device *csdev, void *data) +{ + int ret = 0; + unsigned long flags; + struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + spin_lock_irqsave(&drvdata->spinlock, flags); + + /* No need to continue if the component is already in use. */ + if (drvdata->mode != CS_MODE_DISABLED) { + ret = -EBUSY; + goto out; + } + + /* + * We don't have an internal state to clean up if we fail to setup + * the perf buffer. So we can perform the step before we turn the + * ETB on and leave without cleaning up. + */ + ret = etb_set_buffer(csdev, (struct perf_output_handle *)data); + if (ret) + goto out; + + ret = etb_enable_hw(drvdata); + if (!ret) + drvdata->mode = CS_MODE_PERF; out: - dev_info(drvdata->dev, "ETB enabled\n"); + spin_unlock_irqrestore(&drvdata->spinlock, flags); + return ret; +} + +static int etb_enable(struct coresight_device *csdev, u32 mode, void *data) +{ + int ret; + struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + switch (mode) { + case CS_MODE_SYSFS: + ret = etb_enable_sysfs(csdev); + break; + case CS_MODE_PERF: + ret = etb_enable_perf(csdev, data); + break; + default: + ret = -EINVAL; + break; + } + + if (ret) + return ret; + + dev_dbg(drvdata->dev, "ETB enabled\n"); return 0; } @@ -256,13 +319,16 @@ static void etb_disable(struct coresight_device *csdev) unsigned long flags; spin_lock_irqsave(&drvdata->spinlock, flags); - etb_disable_hw(drvdata); - etb_dump_hw(drvdata); - spin_unlock_irqrestore(&drvdata->spinlock, flags); - local_set(&drvdata->mode, CS_MODE_DISABLED); + /* Disable the ETB only if it needs to */ + if (drvdata->mode != CS_MODE_DISABLED) { + etb_disable_hw(drvdata); + etb_dump_hw(drvdata); + drvdata->mode = CS_MODE_DISABLED; + } + spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "ETB disabled\n"); + dev_dbg(drvdata->dev, "ETB disabled\n"); } static void *etb_alloc_buffer(struct coresight_device *csdev, int cpu, @@ -294,12 +360,14 @@ static void etb_free_buffer(void *config) } static int etb_set_buffer(struct coresight_device *csdev, - struct perf_output_handle *handle, - void *sink_config) + struct perf_output_handle *handle) { int ret = 0; unsigned long head; - struct cs_buffers *buf = sink_config; + struct cs_buffers *buf = etm_perf_sink_config(handle); + + if (!buf) + return -EINVAL; /* wrap head around to the amount of space we have */ head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1); @@ -315,37 +383,7 @@ static int etb_set_buffer(struct coresight_device *csdev, return ret; } -static unsigned long etb_reset_buffer(struct coresight_device *csdev, - struct perf_output_handle *handle, - void *sink_config) -{ - unsigned long size = 0; - struct cs_buffers *buf = sink_config; - - if (buf) { - /* - * In snapshot mode ->data_size holds the new address of the - * ring buffer's head. The size itself is the whole address - * range since we want the latest information. - */ - if (buf->snapshot) - handle->head = local_xchg(&buf->data_size, - buf->nr_pages << PAGE_SHIFT); - - /* - * Tell the tracer PMU how much we got in this run and if - * something went wrong along the way. Nobody else can use - * this cs_buffers instance until we are done. As such - * resetting parameters here and squaring off with the ring - * buffer API in the tracer PMU is fine. - */ - size = local_xchg(&buf->data_size, 0); - } - - return size; -} - -static void etb_update_buffer(struct coresight_device *csdev, +static unsigned long etb_update_buffer(struct coresight_device *csdev, struct perf_output_handle *handle, void *sink_config) { @@ -354,13 +392,13 @@ static void etb_update_buffer(struct coresight_device *csdev, u8 *buf_ptr; const u32 *barrier; u32 read_ptr, write_ptr, capacity; - u32 status, read_data, to_read; - unsigned long offset; + u32 status, read_data; + unsigned long offset, to_read; struct cs_buffers *buf = sink_config; struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); if (!buf) - return; + return 0; capacity = drvdata->buffer_depth * ETB_FRAME_SIZE_WORDS; @@ -465,18 +503,17 @@ static void etb_update_buffer(struct coresight_device *csdev, writel_relaxed(0x0, drvdata->base + ETB_RAM_WRITE_POINTER); /* - * In snapshot mode all we have to do is communicate to - * perf_aux_output_end() the address of the current head. In full - * trace mode the same function expects a size to move rb->aux_head - * forward. + * In snapshot mode we have to update the handle->head to point + * to the new location. */ - if (buf->snapshot) - local_set(&buf->data_size, (cur * PAGE_SIZE) + offset); - else - local_add(to_read, &buf->data_size); - + if (buf->snapshot) { + handle->head = (cur * PAGE_SIZE) + offset; + to_read = buf->nr_pages << PAGE_SHIFT; + } etb_enable_hw(drvdata); CS_LOCK(drvdata->base); + + return to_read; } static const struct coresight_ops_sink etb_sink_ops = { @@ -484,8 +521,6 @@ static const struct coresight_ops_sink etb_sink_ops = { .disable = etb_disable, .alloc_buffer = etb_alloc_buffer, .free_buffer = etb_free_buffer, - .set_buffer = etb_set_buffer, - .reset_buffer = etb_reset_buffer, .update_buffer = etb_update_buffer, }; @@ -498,14 +533,14 @@ static void etb_dump(struct etb_drvdata *drvdata) unsigned long flags; spin_lock_irqsave(&drvdata->spinlock, flags); - if (local_read(&drvdata->mode) == CS_MODE_SYSFS) { + if (drvdata->mode == CS_MODE_SYSFS) { etb_disable_hw(drvdata); etb_dump_hw(drvdata); etb_enable_hw(drvdata); } spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "ETB dumped\n"); + dev_dbg(drvdata->dev, "ETB dumped\n"); } static int etb_open(struct inode *inode, struct file *file) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index 677695635211..abe8249b893b 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -12,6 +12,7 @@ #include <linux/mm.h> #include <linux/init.h> #include <linux/perf_event.h> +#include <linux/percpu-defs.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/workqueue.h> @@ -22,20 +23,6 @@ static struct pmu etm_pmu; static bool etm_perf_up; -/** - * struct etm_event_data - Coresight specifics associated to an event - * @work: Handle to free allocated memory outside IRQ context. - * @mask: Hold the CPU(s) this event was set for. - * @snk_config: The sink configuration. - * @path: An array of path, each slot for one CPU. - */ -struct etm_event_data { - struct work_struct work; - cpumask_t mask; - void *snk_config; - struct list_head **path; -}; - static DEFINE_PER_CPU(struct perf_output_handle, ctx_handle); static DEFINE_PER_CPU(struct coresight_device *, csdev_src); @@ -61,6 +48,18 @@ static const struct attribute_group *etm_pmu_attr_groups[] = { NULL, }; +static inline struct list_head ** +etm_event_cpu_path_ptr(struct etm_event_data *data, int cpu) +{ + return per_cpu_ptr(data->path, cpu); +} + +static inline struct list_head * +etm_event_cpu_path(struct etm_event_data *data, int cpu) +{ + return *etm_event_cpu_path_ptr(data, cpu); +} + static void etm_event_read(struct perf_event *event) {} static int etm_addr_filters_alloc(struct perf_event *event) @@ -114,29 +113,30 @@ static void free_event_data(struct work_struct *work) event_data = container_of(work, struct etm_event_data, work); mask = &event_data->mask; - /* - * First deal with the sink configuration. See comment in - * etm_setup_aux() about why we take the first available path. - */ - if (event_data->snk_config) { + + /* Free the sink buffers, if there are any */ + if (event_data->snk_config && !WARN_ON(cpumask_empty(mask))) { cpu = cpumask_first(mask); - sink = coresight_get_sink(event_data->path[cpu]); + sink = coresight_get_sink(etm_event_cpu_path(event_data, cpu)); if (sink_ops(sink)->free_buffer) sink_ops(sink)->free_buffer(event_data->snk_config); } for_each_cpu(cpu, mask) { - if (!(IS_ERR_OR_NULL(event_data->path[cpu]))) - coresight_release_path(event_data->path[cpu]); + struct list_head **ppath; + + ppath = etm_event_cpu_path_ptr(event_data, cpu); + if (!(IS_ERR_OR_NULL(*ppath))) + coresight_release_path(*ppath); + *ppath = NULL; } - kfree(event_data->path); + free_percpu(event_data->path); kfree(event_data); } static void *alloc_event_data(int cpu) { - int size; cpumask_t *mask; struct etm_event_data *event_data; @@ -145,16 +145,12 @@ static void *alloc_event_data(int cpu) if (!event_data) return NULL; - /* Make sure nothing disappears under us */ - get_online_cpus(); - size = num_online_cpus(); mask = &event_data->mask; if (cpu != -1) cpumask_set_cpu(cpu, mask); else - cpumask_copy(mask, cpu_online_mask); - put_online_cpus(); + cpumask_copy(mask, cpu_present_mask); /* * Each CPU has a single path between source and destination. As such @@ -164,8 +160,8 @@ static void *alloc_event_data(int cpu) * unused memory when dealing with single CPU trace scenarios is small * compared to the cost of searching through an optimized array. */ - event_data->path = kcalloc(size, - sizeof(struct list_head *), GFP_KERNEL); + event_data->path = alloc_percpu(struct list_head *); + if (!event_data->path) { kfree(event_data); return NULL; @@ -206,34 +202,53 @@ static void *etm_setup_aux(int event_cpu, void **pages, * on the cmd line. As such the "enable_sink" flag in sysFS is reset. */ sink = coresight_get_enabled_sink(true); - if (!sink) + if (!sink || !sink_ops(sink)->alloc_buffer) goto err; mask = &event_data->mask; - /* Setup the path for each CPU in a trace session */ + /* + * Setup the path for each CPU in a trace session. We try to build + * trace path for each CPU in the mask. If we don't find an ETM + * for the CPU or fail to build a path, we clear the CPU from the + * mask and continue with the rest. If ever we try to trace on those + * CPUs, we can handle it and fail the session. + */ for_each_cpu(cpu, mask) { + struct list_head *path; struct coresight_device *csdev; csdev = per_cpu(csdev_src, cpu); - if (!csdev) - goto err; + /* + * If there is no ETM associated with this CPU clear it from + * the mask and continue with the rest. If ever we try to trace + * on this CPU, we handle it accordingly. + */ + if (!csdev) { + cpumask_clear_cpu(cpu, mask); + continue; + } /* * Building a path doesn't enable it, it simply builds a * list of devices from source to sink that can be * referenced later when the path is actually needed. */ - event_data->path[cpu] = coresight_build_path(csdev, sink); - if (IS_ERR(event_data->path[cpu])) - goto err; + path = coresight_build_path(csdev, sink); + if (IS_ERR(path)) { + cpumask_clear_cpu(cpu, mask); + continue; + } + + *etm_event_cpu_path_ptr(event_data, cpu) = path; } - if (!sink_ops(sink)->alloc_buffer) + /* If we don't have any CPUs ready for tracing, abort */ + cpu = cpumask_first(mask); + if (cpu >= nr_cpu_ids) goto err; - cpu = cpumask_first(mask); - /* Get the AUX specific data from the sink buffer */ + /* Allocate the sink buffer for this session */ event_data->snk_config = sink_ops(sink)->alloc_buffer(sink, cpu, pages, nr_pages, overwrite); @@ -255,6 +270,7 @@ static void etm_event_start(struct perf_event *event, int flags) struct etm_event_data *event_data; struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); + struct list_head *path; if (!csdev) goto fail; @@ -267,18 +283,14 @@ static void etm_event_start(struct perf_event *event, int flags) if (!event_data) goto fail; + path = etm_event_cpu_path(event_data, cpu); /* We need a sink, no need to continue without one */ - sink = coresight_get_sink(event_data->path[cpu]); - if (WARN_ON_ONCE(!sink || !sink_ops(sink)->set_buffer)) - goto fail_end_stop; - - /* Configure the sink */ - if (sink_ops(sink)->set_buffer(sink, handle, - event_data->snk_config)) + sink = coresight_get_sink(path); + if (WARN_ON_ONCE(!sink)) goto fail_end_stop; /* Nothing will happen without a path */ - if (coresight_enable_path(event_data->path[cpu], CS_MODE_PERF)) + if (coresight_enable_path(path, CS_MODE_PERF, handle)) goto fail_end_stop; /* Tell the perf core the event is alive */ @@ -286,11 +298,13 @@ static void etm_event_start(struct perf_event *event, int flags) /* Finally enable the tracer */ if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF)) - goto fail_end_stop; + goto fail_disable_path; out: return; +fail_disable_path: + coresight_disable_path(path); fail_end_stop: perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); perf_aux_output_end(handle, 0); @@ -306,6 +320,7 @@ static void etm_event_stop(struct perf_event *event, int mode) struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); struct etm_event_data *event_data = perf_get_aux(handle); + struct list_head *path; if (event->hw.state == PERF_HES_STOPPED) return; @@ -313,7 +328,11 @@ static void etm_event_stop(struct perf_event *event, int mode) if (!csdev) return; - sink = coresight_get_sink(event_data->path[cpu]); + path = etm_event_cpu_path(event_data, cpu); + if (!path) + return; + + sink = coresight_get_sink(path); if (!sink) return; @@ -331,20 +350,13 @@ static void etm_event_stop(struct perf_event *event, int mode) if (!sink_ops(sink)->update_buffer) return; - sink_ops(sink)->update_buffer(sink, handle, + size = sink_ops(sink)->update_buffer(sink, handle, event_data->snk_config); - - if (!sink_ops(sink)->reset_buffer) - return; - - size = sink_ops(sink)->reset_buffer(sink, handle, - event_data->snk_config); - perf_aux_output_end(handle, size); } /* Disabling the path make its elements available to other sessions */ - coresight_disable_path(event_data->path[cpu]); + coresight_disable_path(path); } static int etm_event_add(struct perf_event *event, int mode) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.h b/drivers/hwtracing/coresight/coresight-etm-perf.h index 4197df4faf5e..da7d9336a15c 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.h +++ b/drivers/hwtracing/coresight/coresight-etm-perf.h @@ -7,6 +7,7 @@ #ifndef _CORESIGHT_ETM_PERF_H #define _CORESIGHT_ETM_PERF_H +#include <linux/percpu-defs.h> #include "coresight-priv.h" struct coresight_device; @@ -42,14 +43,39 @@ struct etm_filters { bool ssstatus; }; +/** + * struct etm_event_data - Coresight specifics associated to an event + * @work: Handle to free allocated memory outside IRQ context. + * @mask: Hold the CPU(s) this event was set for. + * @snk_config: The sink configuration. + * @path: An array of path, each slot for one CPU. + */ +struct etm_event_data { + struct work_struct work; + cpumask_t mask; + void *snk_config; + struct list_head * __percpu *path; +}; #ifdef CONFIG_CORESIGHT int etm_perf_symlink(struct coresight_device *csdev, bool link); +static inline void *etm_perf_sink_config(struct perf_output_handle *handle) +{ + struct etm_event_data *data = perf_get_aux(handle); + if (data) + return data->snk_config; + return NULL; +} #else static inline int etm_perf_symlink(struct coresight_device *csdev, bool link) { return -EINVAL; } +static inline void *etm_perf_sink_config(struct perf_output_handle *handle) +{ + return NULL; +} + #endif /* CONFIG_CORESIGHT */ #endif diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c index 7c74263c333d..fd5c4cca7db5 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x.c +++ b/drivers/hwtracing/coresight/coresight-etm3x.c @@ -355,11 +355,10 @@ static int etm_parse_event_config(struct etm_drvdata *drvdata, return 0; } -static void etm_enable_hw(void *info) +static int etm_enable_hw(struct etm_drvdata *drvdata) { - int i; + int i, rc; u32 etmcr; - struct etm_drvdata *drvdata = info; struct etm_config *config = &drvdata->config; CS_UNLOCK(drvdata->base); @@ -370,6 +369,9 @@ static void etm_enable_hw(void *info) etm_set_pwrup(drvdata); /* Make sure all registers are accessible */ etm_os_unlock(drvdata); + rc = coresight_claim_device_unlocked(drvdata->base); + if (rc) + goto done; etm_set_prog(drvdata); @@ -418,9 +420,29 @@ static void etm_enable_hw(void *info) etm_writel(drvdata, 0x0, ETMVMIDCVR); etm_clr_prog(drvdata); + +done: + if (rc) + etm_set_pwrdwn(drvdata); CS_LOCK(drvdata->base); - dev_dbg(drvdata->dev, "cpu: %d enable smp call done\n", drvdata->cpu); + dev_dbg(drvdata->dev, "cpu: %d enable smp call done: %d\n", + drvdata->cpu, rc); + return rc; +} + +struct etm_enable_arg { + struct etm_drvdata *drvdata; + int rc; +}; + +static void etm_enable_hw_smp_call(void *info) +{ + struct etm_enable_arg *arg = info; + + if (WARN_ON(!arg)) + return; + arg->rc = etm_enable_hw(arg->drvdata); } static int etm_cpu_id(struct coresight_device *csdev) @@ -475,14 +497,13 @@ static int etm_enable_perf(struct coresight_device *csdev, /* Configure the tracer based on the session's specifics */ etm_parse_event_config(drvdata, event); /* And enable it */ - etm_enable_hw(drvdata); - - return 0; + return etm_enable_hw(drvdata); } static int etm_enable_sysfs(struct coresight_device *csdev) { struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct etm_enable_arg arg = { 0 }; int ret; spin_lock(&drvdata->spinlock); @@ -492,20 +513,21 @@ static int etm_enable_sysfs(struct coresight_device *csdev) * hw configuration will take place on the local CPU during bring up. */ if (cpu_online(drvdata->cpu)) { + arg.drvdata = drvdata; ret = smp_call_function_single(drvdata->cpu, - etm_enable_hw, drvdata, 1); - if (ret) - goto err; + etm_enable_hw_smp_call, &arg, 1); + if (!ret) + ret = arg.rc; + if (!ret) + drvdata->sticky_enable = true; + } else { + ret = -ENODEV; } - drvdata->sticky_enable = true; spin_unlock(&drvdata->spinlock); - dev_info(drvdata->dev, "ETM tracing enabled\n"); - return 0; - -err: - spin_unlock(&drvdata->spinlock); + if (!ret) + dev_dbg(drvdata->dev, "ETM tracing enabled\n"); return ret; } @@ -555,6 +577,8 @@ static void etm_disable_hw(void *info) for (i = 0; i < drvdata->nr_cntr; i++) config->cntr_val[i] = etm_readl(drvdata, ETMCNTVRn(i)); + coresight_disclaim_device_unlocked(drvdata->base); + etm_set_pwrdwn(drvdata); CS_LOCK(drvdata->base); @@ -604,7 +628,7 @@ static void etm_disable_sysfs(struct coresight_device *csdev) spin_unlock(&drvdata->spinlock); cpus_read_unlock(); - dev_info(drvdata->dev, "ETM tracing disabled\n"); + dev_dbg(drvdata->dev, "ETM tracing disabled\n"); } static void etm_disable(struct coresight_device *csdev, diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c index 1d94ebec027b..53e2fb6e86f6 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.c +++ b/drivers/hwtracing/coresight/coresight-etm4x.c @@ -28,6 +28,7 @@ #include <linux/pm_runtime.h> #include <asm/sections.h> #include <asm/local.h> +#include <asm/virt.h> #include "coresight-etm4x.h" #include "coresight-etm-perf.h" @@ -77,16 +78,24 @@ static int etm4_trace_id(struct coresight_device *csdev) return drvdata->trcid; } -static void etm4_enable_hw(void *info) +struct etm4_enable_arg { + struct etmv4_drvdata *drvdata; + int rc; +}; + +static int etm4_enable_hw(struct etmv4_drvdata *drvdata) { - int i; - struct etmv4_drvdata *drvdata = info; + int i, rc; struct etmv4_config *config = &drvdata->config; CS_UNLOCK(drvdata->base); etm4_os_unlock(drvdata); + rc = coresight_claim_device_unlocked(drvdata->base); + if (rc) + goto done; + /* Disable the trace unit before programming trace registers */ writel_relaxed(0, drvdata->base + TRCPRGCTLR); @@ -174,9 +183,21 @@ static void etm4_enable_hw(void *info) dev_err(drvdata->dev, "timeout while waiting for Idle Trace Status\n"); +done: CS_LOCK(drvdata->base); - dev_dbg(drvdata->dev, "cpu: %d enable smp call done\n", drvdata->cpu); + dev_dbg(drvdata->dev, "cpu: %d enable smp call done: %d\n", + drvdata->cpu, rc); + return rc; +} + +static void etm4_enable_hw_smp_call(void *info) +{ + struct etm4_enable_arg *arg = info; + + if (WARN_ON(!arg)) + return; + arg->rc = etm4_enable_hw(arg->drvdata); } static int etm4_parse_event_config(struct etmv4_drvdata *drvdata, @@ -242,7 +263,7 @@ static int etm4_enable_perf(struct coresight_device *csdev, if (ret) goto out; /* And enable it */ - etm4_enable_hw(drvdata); + ret = etm4_enable_hw(drvdata); out: return ret; @@ -251,6 +272,7 @@ out: static int etm4_enable_sysfs(struct coresight_device *csdev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct etm4_enable_arg arg = { 0 }; int ret; spin_lock(&drvdata->spinlock); @@ -259,19 +281,17 @@ static int etm4_enable_sysfs(struct coresight_device *csdev) * Executing etm4_enable_hw on the cpu whose ETM is being enabled * ensures that register writes occur when cpu is powered. */ + arg.drvdata = drvdata; ret = smp_call_function_single(drvdata->cpu, - etm4_enable_hw, drvdata, 1); - if (ret) - goto err; - - drvdata->sticky_enable = true; + etm4_enable_hw_smp_call, &arg, 1); + if (!ret) + ret = arg.rc; + if (!ret) + drvdata->sticky_enable = true; spin_unlock(&drvdata->spinlock); - dev_info(drvdata->dev, "ETM tracing enabled\n"); - return 0; - -err: - spin_unlock(&drvdata->spinlock); + if (!ret) + dev_dbg(drvdata->dev, "ETM tracing enabled\n"); return ret; } @@ -328,6 +348,8 @@ static void etm4_disable_hw(void *info) isb(); writel_relaxed(control, drvdata->base + TRCPRGCTLR); + coresight_disclaim_device_unlocked(drvdata->base); + CS_LOCK(drvdata->base); dev_dbg(drvdata->dev, "cpu: %d disable smp call done\n", drvdata->cpu); @@ -380,7 +402,7 @@ static void etm4_disable_sysfs(struct coresight_device *csdev) spin_unlock(&drvdata->spinlock); cpus_read_unlock(); - dev_info(drvdata->dev, "ETM tracing disabled\n"); + dev_dbg(drvdata->dev, "ETM tracing disabled\n"); } static void etm4_disable(struct coresight_device *csdev, @@ -605,7 +627,7 @@ static void etm4_set_default_config(struct etmv4_config *config) config->vinst_ctrl |= BIT(0); } -static u64 etm4_get_access_type(struct etmv4_config *config) +static u64 etm4_get_ns_access_type(struct etmv4_config *config) { u64 access_type = 0; @@ -616,17 +638,26 @@ static u64 etm4_get_access_type(struct etmv4_config *config) * Bit[13] Exception level 1 - OS * Bit[14] Exception level 2 - Hypervisor * Bit[15] Never implemented - * - * Always stay away from hypervisor mode. */ - access_type = ETM_EXLEVEL_NS_HYP; - - if (config->mode & ETM_MODE_EXCL_KERN) - access_type |= ETM_EXLEVEL_NS_OS; + if (!is_kernel_in_hyp_mode()) { + /* Stay away from hypervisor mode for non-VHE */ + access_type = ETM_EXLEVEL_NS_HYP; + if (config->mode & ETM_MODE_EXCL_KERN) + access_type |= ETM_EXLEVEL_NS_OS; + } else if (config->mode & ETM_MODE_EXCL_KERN) { + access_type = ETM_EXLEVEL_NS_HYP; + } if (config->mode & ETM_MODE_EXCL_USER) access_type |= ETM_EXLEVEL_NS_APP; + return access_type; +} + +static u64 etm4_get_access_type(struct etmv4_config *config) +{ + u64 access_type = etm4_get_ns_access_type(config); + /* * EXLEVEL_S, bits[11:8], don't trace anything happening * in secure state. @@ -880,20 +911,10 @@ void etm4_config_trace_mode(struct etmv4_config *config) addr_acc = config->addr_acc[ETM_DEFAULT_ADDR_COMP]; /* clear default config */ - addr_acc &= ~(ETM_EXLEVEL_NS_APP | ETM_EXLEVEL_NS_OS); + addr_acc &= ~(ETM_EXLEVEL_NS_APP | ETM_EXLEVEL_NS_OS | + ETM_EXLEVEL_NS_HYP); - /* - * EXLEVEL_NS, bits[15:12] - * The Exception levels are: - * Bit[12] Exception level 0 - Application - * Bit[13] Exception level 1 - OS - * Bit[14] Exception level 2 - Hypervisor - * Bit[15] Never implemented - */ - if (mode & ETM_MODE_EXCL_KERN) - addr_acc |= ETM_EXLEVEL_NS_OS; - else - addr_acc |= ETM_EXLEVEL_NS_APP; + addr_acc |= etm4_get_ns_access_type(config); config->addr_acc[ETM_DEFAULT_ADDR_COMP] = addr_acc; config->addr_acc[ETM_DEFAULT_ADDR_COMP + 1] = addr_acc; diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c index 448145a36675..927925151509 100644 --- a/drivers/hwtracing/coresight/coresight-funnel.c +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -25,6 +25,7 @@ #define FUNNEL_HOLDTIME_MASK 0xf00 #define FUNNEL_HOLDTIME_SHFT 0x8 #define FUNNEL_HOLDTIME (0x7 << FUNNEL_HOLDTIME_SHFT) +#define FUNNEL_ENSx_MASK 0xff /** * struct funnel_drvdata - specifics associated to a funnel component @@ -42,31 +43,42 @@ struct funnel_drvdata { unsigned long priority; }; -static void funnel_enable_hw(struct funnel_drvdata *drvdata, int port) +static int funnel_enable_hw(struct funnel_drvdata *drvdata, int port) { u32 functl; + int rc = 0; CS_UNLOCK(drvdata->base); functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); + /* Claim the device only when we enable the first slave */ + if (!(functl & FUNNEL_ENSx_MASK)) { + rc = coresight_claim_device_unlocked(drvdata->base); + if (rc) + goto done; + } + functl &= ~FUNNEL_HOLDTIME_MASK; functl |= FUNNEL_HOLDTIME; functl |= (1 << port); writel_relaxed(functl, drvdata->base + FUNNEL_FUNCTL); writel_relaxed(drvdata->priority, drvdata->base + FUNNEL_PRICTL); - +done: CS_LOCK(drvdata->base); + return rc; } static int funnel_enable(struct coresight_device *csdev, int inport, int outport) { + int rc; struct funnel_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - funnel_enable_hw(drvdata, inport); + rc = funnel_enable_hw(drvdata, inport); - dev_info(drvdata->dev, "FUNNEL inport %d enabled\n", inport); - return 0; + if (!rc) + dev_dbg(drvdata->dev, "FUNNEL inport %d enabled\n", inport); + return rc; } static void funnel_disable_hw(struct funnel_drvdata *drvdata, int inport) @@ -79,6 +91,10 @@ static void funnel_disable_hw(struct funnel_drvdata *drvdata, int inport) functl &= ~(1 << inport); writel_relaxed(functl, drvdata->base + FUNNEL_FUNCTL); + /* Disclaim the device if none of the slaves are now active */ + if (!(functl & FUNNEL_ENSx_MASK)) + coresight_disclaim_device_unlocked(drvdata->base); + CS_LOCK(drvdata->base); } @@ -89,7 +105,7 @@ static void funnel_disable(struct coresight_device *csdev, int inport, funnel_disable_hw(drvdata, inport); - dev_info(drvdata->dev, "FUNNEL inport %d disabled\n", inport); + dev_dbg(drvdata->dev, "FUNNEL inport %d disabled\n", inport); } static const struct coresight_ops_link funnel_link_ops = { diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index 1a6cf3589866..579f34943bf1 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -25,6 +25,13 @@ #define CORESIGHT_DEVID 0xfc8 #define CORESIGHT_DEVTYPE 0xfcc + +/* + * Coresight device CLAIM protocol. + * See PSCI - ARM DEN 0022D, Section: 6.8.1 Debug and Trace save and restore. + */ +#define CORESIGHT_CLAIM_SELF_HOSTED BIT(1) + #define TIMEOUT_US 100 #define BMVAL(val, lsb, msb) ((val & GENMASK(msb, lsb)) >> lsb) @@ -137,7 +144,7 @@ static inline void coresight_write_reg_pair(void __iomem *addr, u64 val, } void coresight_disable_path(struct list_head *path); -int coresight_enable_path(struct list_head *path, u32 mode); +int coresight_enable_path(struct list_head *path, u32 mode, void *sink_data); struct coresight_device *coresight_get_sink(struct list_head *path); struct coresight_device *coresight_get_enabled_sink(bool reset); struct list_head *coresight_build_path(struct coresight_device *csdev, diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index 8d2eaaab6c2f..feac98315471 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -35,7 +35,7 @@ static int replicator_enable(struct coresight_device *csdev, int inport, { struct replicator_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - dev_info(drvdata->dev, "REPLICATOR enabled\n"); + dev_dbg(drvdata->dev, "REPLICATOR enabled\n"); return 0; } @@ -44,7 +44,7 @@ static void replicator_disable(struct coresight_device *csdev, int inport, { struct replicator_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - dev_info(drvdata->dev, "REPLICATOR disabled\n"); + dev_dbg(drvdata->dev, "REPLICATOR disabled\n"); } static const struct coresight_ops_link replicator_link_ops = { diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index c46c70aec1d5..35d6f9709274 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -211,7 +211,7 @@ static int stm_enable(struct coresight_device *csdev, stm_enable_hw(drvdata); spin_unlock(&drvdata->spinlock); - dev_info(drvdata->dev, "STM tracing enabled\n"); + dev_dbg(drvdata->dev, "STM tracing enabled\n"); return 0; } @@ -274,7 +274,7 @@ static void stm_disable(struct coresight_device *csdev, pm_runtime_put(drvdata->dev); local_set(&drvdata->mode, CS_MODE_DISABLED); - dev_info(drvdata->dev, "STM tracing disabled\n"); + dev_dbg(drvdata->dev, "STM tracing disabled\n"); } } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 0549249f4b39..53fc83b72a49 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -10,8 +10,12 @@ #include <linux/slab.h> #include "coresight-priv.h" #include "coresight-tmc.h" +#include "coresight-etm-perf.h" -static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +static int tmc_set_etf_buffer(struct coresight_device *csdev, + struct perf_output_handle *handle); + +static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) { CS_UNLOCK(drvdata->base); @@ -30,33 +34,41 @@ static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata) CS_LOCK(drvdata->base); } +static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = coresight_claim_device(drvdata->base); + + if (rc) + return rc; + + __tmc_etb_enable_hw(drvdata); + return 0; +} + static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata) { char *bufp; u32 read_data, lost; - int i; /* Check if the buffer wrapped around. */ lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL; bufp = drvdata->buf; drvdata->len = 0; while (1) { - for (i = 0; i < drvdata->memwidth; i++) { - read_data = readl_relaxed(drvdata->base + TMC_RRD); - if (read_data == 0xFFFFFFFF) - goto done; - memcpy(bufp, &read_data, 4); - bufp += 4; - drvdata->len += 4; - } + read_data = readl_relaxed(drvdata->base + TMC_RRD); + if (read_data == 0xFFFFFFFF) + break; + memcpy(bufp, &read_data, 4); + bufp += 4; + drvdata->len += 4; } -done: + if (lost) coresight_insert_barrier_packet(drvdata->buf); return; } -static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) +static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) { CS_UNLOCK(drvdata->base); @@ -72,7 +84,13 @@ static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) CS_LOCK(drvdata->base); } -static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) +{ + coresight_disclaim_device(drvdata); + __tmc_etb_disable_hw(drvdata); +} + +static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) { CS_UNLOCK(drvdata->base); @@ -88,13 +106,24 @@ static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata) CS_LOCK(drvdata->base); } +static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = coresight_claim_device(drvdata->base); + + if (rc) + return rc; + + __tmc_etf_enable_hw(drvdata); + return 0; +} + static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) { CS_UNLOCK(drvdata->base); tmc_flush_and_stop(drvdata); tmc_disable_hw(drvdata); - + coresight_disclaim_device_unlocked(drvdata->base); CS_LOCK(drvdata->base); } @@ -170,8 +199,12 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) drvdata->buf = buf; } - drvdata->mode = CS_MODE_SYSFS; - tmc_etb_enable_hw(drvdata); + ret = tmc_etb_enable_hw(drvdata); + if (!ret) + drvdata->mode = CS_MODE_SYSFS; + else + /* Free up the buffer if we failed to enable */ + used = false; out: spin_unlock_irqrestore(&drvdata->spinlock, flags); @@ -182,37 +215,40 @@ out: return ret; } -static int tmc_enable_etf_sink_perf(struct coresight_device *csdev) +static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) { int ret = 0; unsigned long flags; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct perf_output_handle *handle = data; spin_lock_irqsave(&drvdata->spinlock, flags); - if (drvdata->reading) { + do { ret = -EINVAL; - goto out; - } - - /* - * In Perf mode there can be only one writer per sink. There - * is also no need to continue if the ETB/ETR is already operated - * from sysFS. - */ - if (drvdata->mode != CS_MODE_DISABLED) { - ret = -EINVAL; - goto out; - } + if (drvdata->reading) + break; + /* + * In Perf mode there can be only one writer per sink. There + * is also no need to continue if the ETB/ETF is already + * operated from sysFS. + */ + if (drvdata->mode != CS_MODE_DISABLED) + break; - drvdata->mode = CS_MODE_PERF; - tmc_etb_enable_hw(drvdata); -out: + ret = tmc_set_etf_buffer(csdev, handle); + if (ret) + break; + ret = tmc_etb_enable_hw(drvdata); + if (!ret) + drvdata->mode = CS_MODE_PERF; + } while (0); spin_unlock_irqrestore(&drvdata->spinlock, flags); return ret; } -static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode) +static int tmc_enable_etf_sink(struct coresight_device *csdev, + u32 mode, void *data) { int ret; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); @@ -222,7 +258,7 @@ static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode) ret = tmc_enable_etf_sink_sysfs(csdev); break; case CS_MODE_PERF: - ret = tmc_enable_etf_sink_perf(csdev); + ret = tmc_enable_etf_sink_perf(csdev, data); break; /* We shouldn't be here */ default: @@ -233,7 +269,7 @@ static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode) if (ret) return ret; - dev_info(drvdata->dev, "TMC-ETB/ETF enabled\n"); + dev_dbg(drvdata->dev, "TMC-ETB/ETF enabled\n"); return 0; } @@ -256,12 +292,13 @@ static void tmc_disable_etf_sink(struct coresight_device *csdev) spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "TMC-ETB/ETF disabled\n"); + dev_dbg(drvdata->dev, "TMC-ETB/ETF disabled\n"); } static int tmc_enable_etf_link(struct coresight_device *csdev, int inport, int outport) { + int ret; unsigned long flags; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); @@ -271,12 +308,14 @@ static int tmc_enable_etf_link(struct coresight_device *csdev, return -EBUSY; } - tmc_etf_enable_hw(drvdata); - drvdata->mode = CS_MODE_SYSFS; + ret = tmc_etf_enable_hw(drvdata); + if (!ret) + drvdata->mode = CS_MODE_SYSFS; spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "TMC-ETF enabled\n"); - return 0; + if (!ret) + dev_dbg(drvdata->dev, "TMC-ETF enabled\n"); + return ret; } static void tmc_disable_etf_link(struct coresight_device *csdev, @@ -295,7 +334,7 @@ static void tmc_disable_etf_link(struct coresight_device *csdev, drvdata->mode = CS_MODE_DISABLED; spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "TMC-ETF disabled\n"); + dev_dbg(drvdata->dev, "TMC-ETF disabled\n"); } static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, int cpu, @@ -328,12 +367,14 @@ static void tmc_free_etf_buffer(void *config) } static int tmc_set_etf_buffer(struct coresight_device *csdev, - struct perf_output_handle *handle, - void *sink_config) + struct perf_output_handle *handle) { int ret = 0; unsigned long head; - struct cs_buffers *buf = sink_config; + struct cs_buffers *buf = etm_perf_sink_config(handle); + + if (!buf) + return -EINVAL; /* wrap head around to the amount of space we have */ head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1); @@ -349,36 +390,7 @@ static int tmc_set_etf_buffer(struct coresight_device *csdev, return ret; } -static unsigned long tmc_reset_etf_buffer(struct coresight_device *csdev, - struct perf_output_handle *handle, - void *sink_config) -{ - long size = 0; - struct cs_buffers *buf = sink_config; - - if (buf) { - /* - * In snapshot mode ->data_size holds the new address of the - * ring buffer's head. The size itself is the whole address - * range since we want the latest information. - */ - if (buf->snapshot) - handle->head = local_xchg(&buf->data_size, - buf->nr_pages << PAGE_SHIFT); - /* - * Tell the tracer PMU how much we got in this run and if - * something went wrong along the way. Nobody else can use - * this cs_buffers instance until we are done. As such - * resetting parameters here and squaring off with the ring - * buffer API in the tracer PMU is fine. - */ - size = local_xchg(&buf->data_size, 0); - } - - return size; -} - -static void tmc_update_etf_buffer(struct coresight_device *csdev, +static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, struct perf_output_handle *handle, void *sink_config) { @@ -387,17 +399,17 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev, const u32 *barrier; u32 *buf_ptr; u64 read_ptr, write_ptr; - u32 status, to_read; - unsigned long offset; + u32 status; + unsigned long offset, to_read; struct cs_buffers *buf = sink_config; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); if (!buf) - return; + return 0; /* This shouldn't happen */ if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF)) - return; + return 0; CS_UNLOCK(drvdata->base); @@ -438,10 +450,10 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev, case TMC_MEM_INTF_WIDTH_32BITS: case TMC_MEM_INTF_WIDTH_64BITS: case TMC_MEM_INTF_WIDTH_128BITS: - mask = GENMASK(31, 5); + mask = GENMASK(31, 4); break; case TMC_MEM_INTF_WIDTH_256BITS: - mask = GENMASK(31, 6); + mask = GENMASK(31, 5); break; } @@ -486,18 +498,14 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev, } } - /* - * In snapshot mode all we have to do is communicate to - * perf_aux_output_end() the address of the current head. In full - * trace mode the same function expects a size to move rb->aux_head - * forward. - */ - if (buf->snapshot) - local_set(&buf->data_size, (cur * PAGE_SIZE) + offset); - else - local_add(to_read, &buf->data_size); - + /* In snapshot mode we have to update the head */ + if (buf->snapshot) { + handle->head = (cur * PAGE_SIZE) + offset; + to_read = buf->nr_pages << PAGE_SHIFT; + } CS_LOCK(drvdata->base); + + return to_read; } static const struct coresight_ops_sink tmc_etf_sink_ops = { @@ -505,8 +513,6 @@ static const struct coresight_ops_sink tmc_etf_sink_ops = { .disable = tmc_disable_etf_sink, .alloc_buffer = tmc_alloc_etf_buffer, .free_buffer = tmc_free_etf_buffer, - .set_buffer = tmc_set_etf_buffer, - .reset_buffer = tmc_reset_etf_buffer, .update_buffer = tmc_update_etf_buffer, }; @@ -563,7 +569,7 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) /* Disable the TMC if need be */ if (drvdata->mode == CS_MODE_SYSFS) - tmc_etb_disable_hw(drvdata); + __tmc_etb_disable_hw(drvdata); drvdata->reading = true; out: @@ -603,7 +609,7 @@ int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) * can't be NULL. */ memset(drvdata->buf, 0, drvdata->size); - tmc_etb_enable_hw(drvdata); + __tmc_etb_enable_hw(drvdata); } else { /* * The ETB/ETF is not tracing and the buffer was just read. diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 2eda5de304c2..f684283890d3 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -10,6 +10,7 @@ #include <linux/slab.h> #include <linux/vmalloc.h> #include "coresight-catu.h" +#include "coresight-etm-perf.h" #include "coresight-priv.h" #include "coresight-tmc.h" @@ -21,6 +22,28 @@ struct etr_flat_buf { }; /* + * etr_perf_buffer - Perf buffer used for ETR + * @etr_buf - Actual buffer used by the ETR + * @snaphost - Perf session mode + * @head - handle->head at the beginning of the session. + * @nr_pages - Number of pages in the ring buffer. + * @pages - Array of Pages in the ring buffer. + */ +struct etr_perf_buffer { + struct etr_buf *etr_buf; + bool snapshot; + unsigned long head; + int nr_pages; + void **pages; +}; + +/* Convert the perf index to an offset within the ETR buffer */ +#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) + +/* Lower limit for ETR hardware buffer */ +#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M + +/* * The TMC ETR SG has a page size of 4K. The SG table contains pointers * to 4KB buffers. However, the OS may use a PAGE_SIZE different from * 4K (i.e, 16KB or 64KB). This implies that a single OS page could @@ -536,7 +559,7 @@ tmc_init_etr_sg_table(struct device *dev, int node, sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages); if (IS_ERR(sg_table)) { kfree(etr_table); - return ERR_PTR(PTR_ERR(sg_table)); + return ERR_CAST(sg_table); } etr_table->sg_table = sg_table; @@ -728,12 +751,14 @@ tmc_etr_get_catu_device(struct tmc_drvdata *drvdata) return NULL; } -static inline void tmc_etr_enable_catu(struct tmc_drvdata *drvdata) +static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata, + struct etr_buf *etr_buf) { struct coresight_device *catu = tmc_etr_get_catu_device(drvdata); if (catu && helper_ops(catu)->enable) - helper_ops(catu)->enable(catu, drvdata->etr_buf); + return helper_ops(catu)->enable(catu, etr_buf); + return 0; } static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata) @@ -895,17 +920,11 @@ static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata) tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset); } -static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata) +static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata) { u32 axictl, sts; struct etr_buf *etr_buf = drvdata->etr_buf; - /* - * If this ETR is connected to a CATU, enable it before we turn - * this on - */ - tmc_etr_enable_catu(drvdata); - CS_UNLOCK(drvdata->base); /* Wait for TMCSReady bit to be set */ @@ -924,11 +943,8 @@ static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata) axictl |= TMC_AXICTL_ARCACHE_OS; } - if (etr_buf->mode == ETR_MODE_ETR_SG) { - if (WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG))) - return; + if (etr_buf->mode == ETR_MODE_ETR_SG) axictl |= TMC_AXICTL_SCT_GAT_MODE; - } writel_relaxed(axictl, drvdata->base + TMC_AXICTL); tmc_write_dba(drvdata, etr_buf->hwaddr); @@ -954,19 +970,54 @@ static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata) CS_LOCK(drvdata->base); } +static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata, + struct etr_buf *etr_buf) +{ + int rc; + + /* Callers should provide an appropriate buffer for use */ + if (WARN_ON(!etr_buf)) + return -EINVAL; + + if ((etr_buf->mode == ETR_MODE_ETR_SG) && + WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG))) + return -EINVAL; + + if (WARN_ON(drvdata->etr_buf)) + return -EBUSY; + + /* + * If this ETR is connected to a CATU, enable it before we turn + * this on. + */ + rc = tmc_etr_enable_catu(drvdata, etr_buf); + if (rc) + return rc; + rc = coresight_claim_device(drvdata->base); + if (!rc) { + drvdata->etr_buf = etr_buf; + __tmc_etr_enable_hw(drvdata); + } + + return rc; +} + /* * Return the available trace data in the buffer (starts at etr_buf->offset, * limited by etr_buf->len) from @pos, with a maximum limit of @len, * also updating the @bufpp on where to find it. Since the trace data * starts at anywhere in the buffer, depending on the RRP, we adjust the * @len returned to handle buffer wrapping around. + * + * We are protected here by drvdata->reading != 0, which ensures the + * sysfs_buf stays alive. */ ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata, loff_t pos, size_t len, char **bufpp) { s64 offset; ssize_t actual = len; - struct etr_buf *etr_buf = drvdata->etr_buf; + struct etr_buf *etr_buf = drvdata->sysfs_buf; if (pos + actual > etr_buf->len) actual = etr_buf->len - pos; @@ -996,10 +1047,17 @@ tmc_etr_free_sysfs_buf(struct etr_buf *buf) static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata) { - tmc_sync_etr_buf(drvdata); + struct etr_buf *etr_buf = drvdata->etr_buf; + + if (WARN_ON(drvdata->sysfs_buf != etr_buf)) { + tmc_etr_free_sysfs_buf(drvdata->sysfs_buf); + drvdata->sysfs_buf = NULL; + } else { + tmc_sync_etr_buf(drvdata); + } } -static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) +static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata) { CS_UNLOCK(drvdata->base); @@ -1015,8 +1073,16 @@ static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) CS_LOCK(drvdata->base); +} + +static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) +{ + __tmc_etr_disable_hw(drvdata); /* Disable CATU device if this ETR is connected to one */ tmc_etr_disable_catu(drvdata); + coresight_disclaim_device(drvdata->base); + /* Reset the ETR buf used by hardware */ + drvdata->etr_buf = NULL; } static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) @@ -1024,7 +1090,7 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) int ret = 0; unsigned long flags; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - struct etr_buf *new_buf = NULL, *free_buf = NULL; + struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL; /* * If we are enabling the ETR from disabled state, we need to make @@ -1035,7 +1101,8 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) * with the lock released. */ spin_lock_irqsave(&drvdata->spinlock, flags); - if (!drvdata->etr_buf || (drvdata->etr_buf->size != drvdata->size)) { + sysfs_buf = READ_ONCE(drvdata->sysfs_buf); + if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) { spin_unlock_irqrestore(&drvdata->spinlock, flags); /* Allocate memory with the locks released */ @@ -1064,14 +1131,15 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) * If we don't have a buffer or it doesn't match the requested size, * use the buffer allocated above. Otherwise reuse the existing buffer. */ - if (!drvdata->etr_buf || - (new_buf && drvdata->etr_buf->size != new_buf->size)) { - free_buf = drvdata->etr_buf; - drvdata->etr_buf = new_buf; + sysfs_buf = READ_ONCE(drvdata->sysfs_buf); + if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) { + free_buf = sysfs_buf; + drvdata->sysfs_buf = new_buf; } - drvdata->mode = CS_MODE_SYSFS; - tmc_etr_enable_hw(drvdata); + ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf); + if (!ret) + drvdata->mode = CS_MODE_SYSFS; out: spin_unlock_irqrestore(&drvdata->spinlock, flags); @@ -1080,24 +1148,244 @@ out: tmc_etr_free_sysfs_buf(free_buf); if (!ret) - dev_info(drvdata->dev, "TMC-ETR enabled\n"); + dev_dbg(drvdata->dev, "TMC-ETR enabled\n"); return ret; } -static int tmc_enable_etr_sink_perf(struct coresight_device *csdev) +/* + * tmc_etr_setup_perf_buf: Allocate ETR buffer for use by perf. + * The size of the hardware buffer is dependent on the size configured + * via sysfs and the perf ring buffer size. We prefer to allocate the + * largest possible size, scaling down the size by half until it + * reaches a minimum limit (1M), beyond which we give up. + */ +static struct etr_perf_buffer * +tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, int node, int nr_pages, + void **pages, bool snapshot) { - /* We don't support perf mode yet ! */ - return -EINVAL; + struct etr_buf *etr_buf; + struct etr_perf_buffer *etr_perf; + unsigned long size; + + etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node); + if (!etr_perf) + return ERR_PTR(-ENOMEM); + + /* + * Try to match the perf ring buffer size if it is larger + * than the size requested via sysfs. + */ + if ((nr_pages << PAGE_SHIFT) > drvdata->size) { + etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT), + 0, node, NULL); + if (!IS_ERR(etr_buf)) + goto done; + } + + /* + * Else switch to configured size for this ETR + * and scale down until we hit the minimum limit. + */ + size = drvdata->size; + do { + etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL); + if (!IS_ERR(etr_buf)) + goto done; + size /= 2; + } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE); + + kfree(etr_perf); + return ERR_PTR(-ENOMEM); + +done: + etr_perf->etr_buf = etr_buf; + return etr_perf; } -static int tmc_enable_etr_sink(struct coresight_device *csdev, u32 mode) + +static void *tmc_alloc_etr_buffer(struct coresight_device *csdev, + int cpu, void **pages, int nr_pages, + bool snapshot) +{ + struct etr_perf_buffer *etr_perf; + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + if (cpu == -1) + cpu = smp_processor_id(); + + etr_perf = tmc_etr_setup_perf_buf(drvdata, cpu_to_node(cpu), + nr_pages, pages, snapshot); + if (IS_ERR(etr_perf)) { + dev_dbg(drvdata->dev, "Unable to allocate ETR buffer\n"); + return NULL; + } + + etr_perf->snapshot = snapshot; + etr_perf->nr_pages = nr_pages; + etr_perf->pages = pages; + + return etr_perf; +} + +static void tmc_free_etr_buffer(void *config) +{ + struct etr_perf_buffer *etr_perf = config; + + if (etr_perf->etr_buf) + tmc_free_etr_buf(etr_perf->etr_buf); + kfree(etr_perf); +} + +/* + * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware + * buffer to the perf ring buffer. + */ +static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf) +{ + long bytes, to_copy; + long pg_idx, pg_offset, src_offset; + unsigned long head = etr_perf->head; + char **dst_pages, *src_buf; + struct etr_buf *etr_buf = etr_perf->etr_buf; + + head = etr_perf->head; + pg_idx = head >> PAGE_SHIFT; + pg_offset = head & (PAGE_SIZE - 1); + dst_pages = (char **)etr_perf->pages; + src_offset = etr_buf->offset; + to_copy = etr_buf->len; + + while (to_copy > 0) { + /* + * In one iteration, we can copy minimum of : + * 1) what is available in the source buffer, + * 2) what is available in the source buffer, before it + * wraps around. + * 3) what is available in the destination page. + * in one iteration. + */ + bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy, + &src_buf); + if (WARN_ON_ONCE(bytes <= 0)) + break; + bytes = min(bytes, (long)(PAGE_SIZE - pg_offset)); + + memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes); + + to_copy -= bytes; + + /* Move destination pointers */ + pg_offset += bytes; + if (pg_offset == PAGE_SIZE) { + pg_offset = 0; + if (++pg_idx == etr_perf->nr_pages) + pg_idx = 0; + } + + /* Move source pointers */ + src_offset += bytes; + if (src_offset >= etr_buf->size) + src_offset -= etr_buf->size; + } +} + +/* + * tmc_update_etr_buffer : Update the perf ring buffer with the + * available trace data. We use software double buffering at the moment. + * + * TODO: Add support for reusing the perf ring buffer. + */ +static unsigned long +tmc_update_etr_buffer(struct coresight_device *csdev, + struct perf_output_handle *handle, + void *config) +{ + bool lost = false; + unsigned long flags, size = 0; + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct etr_perf_buffer *etr_perf = config; + struct etr_buf *etr_buf = etr_perf->etr_buf; + + spin_lock_irqsave(&drvdata->spinlock, flags); + if (WARN_ON(drvdata->perf_data != etr_perf)) { + lost = true; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + goto out; + } + + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + tmc_sync_etr_buf(drvdata); + + CS_LOCK(drvdata->base); + /* Reset perf specific data */ + drvdata->perf_data = NULL; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + size = etr_buf->len; + tmc_etr_sync_perf_buffer(etr_perf); + + /* + * Update handle->head in snapshot mode. Also update the size to the + * hardware buffer size if there was an overflow. + */ + if (etr_perf->snapshot) { + handle->head += size; + if (etr_buf->full) + size = etr_buf->size; + } + + lost |= etr_buf->full; +out: + if (lost) + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + return size; +} + +static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) +{ + int rc = 0; + unsigned long flags; + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct perf_output_handle *handle = data; + struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle); + + spin_lock_irqsave(&drvdata->spinlock, flags); + /* + * There can be only one writer per sink in perf mode. If the sink + * is already open in SYSFS mode, we can't use it. + */ + if (drvdata->mode != CS_MODE_DISABLED || WARN_ON(drvdata->perf_data)) { + rc = -EBUSY; + goto unlock_out; + } + + if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) { + rc = -EINVAL; + goto unlock_out; + } + + etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf); + drvdata->perf_data = etr_perf; + rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf); + if (!rc) + drvdata->mode = CS_MODE_PERF; + +unlock_out: + spin_unlock_irqrestore(&drvdata->spinlock, flags); + return rc; +} + +static int tmc_enable_etr_sink(struct coresight_device *csdev, + u32 mode, void *data) { switch (mode) { case CS_MODE_SYSFS: return tmc_enable_etr_sink_sysfs(csdev); case CS_MODE_PERF: - return tmc_enable_etr_sink_perf(csdev); + return tmc_enable_etr_sink_perf(csdev, data); } /* We shouldn't be here */ @@ -1123,12 +1411,15 @@ static void tmc_disable_etr_sink(struct coresight_device *csdev) spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "TMC-ETR disabled\n"); + dev_dbg(drvdata->dev, "TMC-ETR disabled\n"); } static const struct coresight_ops_sink tmc_etr_sink_ops = { .enable = tmc_enable_etr_sink, .disable = tmc_disable_etr_sink, + .alloc_buffer = tmc_alloc_etr_buffer, + .update_buffer = tmc_update_etr_buffer, + .free_buffer = tmc_free_etr_buffer, }; const struct coresight_ops tmc_etr_cs_ops = { @@ -1150,21 +1441,19 @@ int tmc_read_prepare_etr(struct tmc_drvdata *drvdata) goto out; } - /* Don't interfere if operated from Perf */ - if (drvdata->mode == CS_MODE_PERF) { - ret = -EINVAL; - goto out; - } - - /* If drvdata::etr_buf is NULL the trace data has been read already */ - if (drvdata->etr_buf == NULL) { + /* + * We can safely allow reads even if the ETR is operating in PERF mode, + * since the sysfs session is captured in mode specific data. + * If drvdata::sysfs_data is NULL the trace data has been read already. + */ + if (!drvdata->sysfs_buf) { ret = -EINVAL; goto out; } - /* Disable the TMC if need be */ + /* Disable the TMC if we are trying to read from a running session. */ if (drvdata->mode == CS_MODE_SYSFS) - tmc_etr_disable_hw(drvdata); + __tmc_etr_disable_hw(drvdata); drvdata->reading = true; out: @@ -1176,7 +1465,7 @@ out: int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata) { unsigned long flags; - struct etr_buf *etr_buf = NULL; + struct etr_buf *sysfs_buf = NULL; /* config types are set a boot time and never change */ if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR)) @@ -1191,22 +1480,22 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata) * buffer. Since the tracer is still enabled drvdata::buf can't * be NULL. */ - tmc_etr_enable_hw(drvdata); + __tmc_etr_enable_hw(drvdata); } else { /* * The ETR is not tracing and the buffer was just read. * As such prepare to free the trace buffer. */ - etr_buf = drvdata->etr_buf; - drvdata->etr_buf = NULL; + sysfs_buf = drvdata->sysfs_buf; + drvdata->sysfs_buf = NULL; } drvdata->reading = false; spin_unlock_irqrestore(&drvdata->spinlock, flags); /* Free allocated memory out side of the spinlock */ - if (etr_buf) - tmc_free_etr_buf(etr_buf); + if (sysfs_buf) + tmc_etr_free_sysfs_buf(sysfs_buf); return 0; } diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c index 1b817ec1192c..ea249f0bcd73 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.c +++ b/drivers/hwtracing/coresight/coresight-tmc.c @@ -81,7 +81,7 @@ static int tmc_read_prepare(struct tmc_drvdata *drvdata) } if (!ret) - dev_info(drvdata->dev, "TMC read start\n"); + dev_dbg(drvdata->dev, "TMC read start\n"); return ret; } @@ -103,7 +103,7 @@ static int tmc_read_unprepare(struct tmc_drvdata *drvdata) } if (!ret) - dev_info(drvdata->dev, "TMC read end\n"); + dev_dbg(drvdata->dev, "TMC read end\n"); return ret; } diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h index 7027bd60c4cc..487c53701e9c 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.h +++ b/drivers/hwtracing/coresight/coresight-tmc.h @@ -170,6 +170,8 @@ struct etr_buf { * @trigger_cntr: amount of words to store after a trigger. * @etr_caps: Bitmask of capabilities of the TMC ETR, inferred from the * device configuration register (DEVID) + * @perf_data: PERF buffer for ETR. + * @sysfs_data: SYSFS buffer for ETR. */ struct tmc_drvdata { void __iomem *base; @@ -189,6 +191,8 @@ struct tmc_drvdata { enum tmc_mem_intf_width memwidth; u32 trigger_cntr; u32 etr_caps; + struct etr_buf *sysfs_buf; + void *perf_data; }; struct etr_buf_operations { diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index 459ef930d98c..b2f72a1fa402 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -68,13 +68,13 @@ static void tpiu_enable_hw(struct tpiu_drvdata *drvdata) CS_LOCK(drvdata->base); } -static int tpiu_enable(struct coresight_device *csdev, u32 mode) +static int tpiu_enable(struct coresight_device *csdev, u32 mode, void *__unused) { struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); tpiu_enable_hw(drvdata); - dev_info(drvdata->dev, "TPIU enabled\n"); + dev_dbg(drvdata->dev, "TPIU enabled\n"); return 0; } @@ -100,7 +100,7 @@ static void tpiu_disable(struct coresight_device *csdev) tpiu_disable_hw(drvdata); - dev_info(drvdata->dev, "TPIU disabled\n"); + dev_dbg(drvdata->dev, "TPIU disabled\n"); } static const struct coresight_ops_sink tpiu_sink_ops = { diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c index 3e07fd335f8c..2b0df1a0a8df 100644 --- a/drivers/hwtracing/coresight/coresight.c +++ b/drivers/hwtracing/coresight/coresight.c @@ -128,16 +128,105 @@ static int coresight_find_link_outport(struct coresight_device *csdev, return -ENODEV; } -static int coresight_enable_sink(struct coresight_device *csdev, u32 mode) +static inline u32 coresight_read_claim_tags(void __iomem *base) +{ + return readl_relaxed(base + CORESIGHT_CLAIMCLR); +} + +static inline bool coresight_is_claimed_self_hosted(void __iomem *base) +{ + return coresight_read_claim_tags(base) == CORESIGHT_CLAIM_SELF_HOSTED; +} + +static inline bool coresight_is_claimed_any(void __iomem *base) +{ + return coresight_read_claim_tags(base) != 0; +} + +static inline void coresight_set_claim_tags(void __iomem *base) +{ + writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMSET); + isb(); +} + +static inline void coresight_clear_claim_tags(void __iomem *base) +{ + writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMCLR); + isb(); +} + +/* + * coresight_claim_device_unlocked : Claim the device for self-hosted usage + * to prevent an external tool from touching this device. As per PSCI + * standards, section "Preserving the execution context" => "Debug and Trace + * save and Restore", DBGCLAIM[1] is reserved for Self-hosted debug/trace and + * DBGCLAIM[0] is reserved for external tools. + * + * Called with CS_UNLOCKed for the component. + * Returns : 0 on success + */ +int coresight_claim_device_unlocked(void __iomem *base) +{ + if (coresight_is_claimed_any(base)) + return -EBUSY; + + coresight_set_claim_tags(base); + if (coresight_is_claimed_self_hosted(base)) + return 0; + /* There was a race setting the tags, clean up and fail */ + coresight_clear_claim_tags(base); + return -EBUSY; +} + +int coresight_claim_device(void __iomem *base) +{ + int rc; + + CS_UNLOCK(base); + rc = coresight_claim_device_unlocked(base); + CS_LOCK(base); + + return rc; +} + +/* + * coresight_disclaim_device_unlocked : Clear the claim tags for the device. + * Called with CS_UNLOCKed for the component. + */ +void coresight_disclaim_device_unlocked(void __iomem *base) +{ + + if (coresight_is_claimed_self_hosted(base)) + coresight_clear_claim_tags(base); + else + /* + * The external agent may have not honoured our claim + * and has manipulated it. Or something else has seriously + * gone wrong in our driver. + */ + WARN_ON_ONCE(1); +} + +void coresight_disclaim_device(void __iomem *base) +{ + CS_UNLOCK(base); + coresight_disclaim_device_unlocked(base); + CS_LOCK(base); +} + +static int coresight_enable_sink(struct coresight_device *csdev, + u32 mode, void *data) { int ret; - if (!csdev->enable) { - if (sink_ops(csdev)->enable) { - ret = sink_ops(csdev)->enable(csdev, mode); - if (ret) - return ret; - } + /* + * We need to make sure the "new" session is compatible with the + * existing "mode" of operation. + */ + if (sink_ops(csdev)->enable) { + ret = sink_ops(csdev)->enable(csdev, mode, data); + if (ret) + return ret; csdev->enable = true; } @@ -184,8 +273,10 @@ static int coresight_enable_link(struct coresight_device *csdev, if (atomic_inc_return(&csdev->refcnt[refport]) == 1) { if (link_ops(csdev)->enable) { ret = link_ops(csdev)->enable(csdev, inport, outport); - if (ret) + if (ret) { + atomic_dec(&csdev->refcnt[refport]); return ret; + } } } @@ -274,13 +365,21 @@ static bool coresight_disable_source(struct coresight_device *csdev) return !csdev->enable; } -void coresight_disable_path(struct list_head *path) +/* + * coresight_disable_path_from : Disable components in the given path beyond + * @nd in the list. If @nd is NULL, all the components, except the SOURCE are + * disabled. + */ +static void coresight_disable_path_from(struct list_head *path, + struct coresight_node *nd) { u32 type; - struct coresight_node *nd; struct coresight_device *csdev, *parent, *child; - list_for_each_entry(nd, path, link) { + if (!nd) + nd = list_first_entry(path, struct coresight_node, link); + + list_for_each_entry_continue(nd, path, link) { csdev = nd->csdev; type = csdev->type; @@ -300,7 +399,12 @@ void coresight_disable_path(struct list_head *path) coresight_disable_sink(csdev); break; case CORESIGHT_DEV_TYPE_SOURCE: - /* sources are disabled from either sysFS or Perf */ + /* + * We skip the first node in the path assuming that it + * is the source. So we don't expect a source device in + * the middle of a path. + */ + WARN_ON(1); break; case CORESIGHT_DEV_TYPE_LINK: parent = list_prev_entry(nd, link)->csdev; @@ -313,7 +417,12 @@ void coresight_disable_path(struct list_head *path) } } -int coresight_enable_path(struct list_head *path, u32 mode) +void coresight_disable_path(struct list_head *path) +{ + coresight_disable_path_from(path, NULL); +} + +int coresight_enable_path(struct list_head *path, u32 mode, void *sink_data) { int ret = 0; @@ -338,9 +447,15 @@ int coresight_enable_path(struct list_head *path, u32 mode) switch (type) { case CORESIGHT_DEV_TYPE_SINK: - ret = coresight_enable_sink(csdev, mode); + ret = coresight_enable_sink(csdev, mode, sink_data); + /* + * Sink is the first component turned on. If we + * failed to enable the sink, there are no components + * that need disabling. Disabling the path here + * would mean we could disrupt an existing session. + */ if (ret) - goto err; + goto out; break; case CORESIGHT_DEV_TYPE_SOURCE: /* sources are enabled from either sysFS or Perf */ @@ -360,7 +475,7 @@ int coresight_enable_path(struct list_head *path, u32 mode) out: return ret; err: - coresight_disable_path(path); + coresight_disable_path_from(path, nd); goto out; } @@ -635,7 +750,7 @@ int coresight_enable(struct coresight_device *csdev) goto out; } - ret = coresight_enable_path(path, CS_MODE_SYSFS); + ret = coresight_enable_path(path, CS_MODE_SYSFS, NULL); if (ret) goto err_path; @@ -995,18 +1110,16 @@ postcore_initcall(coresight_init); struct coresight_device *coresight_register(struct coresight_desc *desc) { - int i; int ret; int link_subtype; int nr_refcnts = 1; atomic_t *refcnts = NULL; struct coresight_device *csdev; - struct coresight_connection *conns = NULL; csdev = kzalloc(sizeof(*csdev), GFP_KERNEL); if (!csdev) { ret = -ENOMEM; - goto err_kzalloc_csdev; + goto err_out; } if (desc->type == CORESIGHT_DEV_TYPE_LINK || @@ -1022,7 +1135,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) refcnts = kcalloc(nr_refcnts, sizeof(*refcnts), GFP_KERNEL); if (!refcnts) { ret = -ENOMEM; - goto err_kzalloc_refcnts; + goto err_free_csdev; } csdev->refcnt = refcnts; @@ -1030,22 +1143,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) csdev->nr_inport = desc->pdata->nr_inport; csdev->nr_outport = desc->pdata->nr_outport; - /* Initialise connections if there is at least one outport */ - if (csdev->nr_outport) { - conns = kcalloc(csdev->nr_outport, sizeof(*conns), GFP_KERNEL); - if (!conns) { - ret = -ENOMEM; - goto err_kzalloc_conns; - } - - for (i = 0; i < csdev->nr_outport; i++) { - conns[i].outport = desc->pdata->outports[i]; - conns[i].child_name = desc->pdata->child_names[i]; - conns[i].child_port = desc->pdata->child_ports[i]; - } - } - - csdev->conns = conns; + csdev->conns = desc->pdata->conns; csdev->type = desc->type; csdev->subtype = desc->subtype; @@ -1062,7 +1160,11 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) ret = device_register(&csdev->dev); if (ret) { put_device(&csdev->dev); - goto err_kzalloc_csdev; + /* + * All resources are free'd explicitly via + * coresight_device_release(), triggered from put_device(). + */ + goto err_out; } mutex_lock(&coresight_mutex); @@ -1074,11 +1176,9 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) return csdev; -err_kzalloc_conns: - kfree(refcnts); -err_kzalloc_refcnts: +err_free_csdev: kfree(csdev); -err_kzalloc_csdev: +err_out: return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(coresight_register); diff --git a/drivers/hwtracing/coresight/of_coresight.c b/drivers/hwtracing/coresight/of_coresight.c index 6880bee195c8..89092f83567e 100644 --- a/drivers/hwtracing/coresight/of_coresight.c +++ b/drivers/hwtracing/coresight/of_coresight.c @@ -45,8 +45,13 @@ of_coresight_get_endpoint_device(struct device_node *endpoint) endpoint, of_dev_node_match); } -static void of_coresight_get_ports(const struct device_node *node, - int *nr_inport, int *nr_outport) +static inline bool of_coresight_legacy_ep_is_input(struct device_node *ep) +{ + return of_property_read_bool(ep, "slave-mode"); +} + +static void of_coresight_get_ports_legacy(const struct device_node *node, + int *nr_inport, int *nr_outport) { struct device_node *ep = NULL; int in = 0, out = 0; @@ -56,7 +61,7 @@ static void of_coresight_get_ports(const struct device_node *node, if (!ep) break; - if (of_property_read_bool(ep, "slave-mode")) + if (of_coresight_legacy_ep_is_input(ep)) in++; else out++; @@ -67,32 +72,77 @@ static void of_coresight_get_ports(const struct device_node *node, *nr_outport = out; } +static struct device_node *of_coresight_get_port_parent(struct device_node *ep) +{ + struct device_node *parent = of_graph_get_port_parent(ep); + + /* + * Skip one-level up to the real device node, if we + * are using the new bindings. + */ + if (!of_node_cmp(parent->name, "in-ports") || + !of_node_cmp(parent->name, "out-ports")) + parent = of_get_next_parent(parent); + + return parent; +} + +static inline struct device_node * +of_coresight_get_input_ports_node(const struct device_node *node) +{ + return of_get_child_by_name(node, "in-ports"); +} + +static inline struct device_node * +of_coresight_get_output_ports_node(const struct device_node *node) +{ + return of_get_child_by_name(node, "out-ports"); +} + +static inline int +of_coresight_count_ports(struct device_node *port_parent) +{ + int i = 0; + struct device_node *ep = NULL; + + while ((ep = of_graph_get_next_endpoint(port_parent, ep))) + i++; + return i; +} + +static void of_coresight_get_ports(const struct device_node *node, + int *nr_inport, int *nr_outport) +{ + struct device_node *input_ports = NULL, *output_ports = NULL; + + input_ports = of_coresight_get_input_ports_node(node); + output_ports = of_coresight_get_output_ports_node(node); + + if (input_ports || output_ports) { + if (input_ports) { + *nr_inport = of_coresight_count_ports(input_ports); + of_node_put(input_ports); + } + if (output_ports) { + *nr_outport = of_coresight_count_ports(output_ports); + of_node_put(output_ports); + } + } else { + /* Fall back to legacy DT bindings parsing */ + of_coresight_get_ports_legacy(node, nr_inport, nr_outport); + } +} + static int of_coresight_alloc_memory(struct device *dev, struct coresight_platform_data *pdata) { - /* List of output port on this component */ - pdata->outports = devm_kcalloc(dev, - pdata->nr_outport, - sizeof(*pdata->outports), - GFP_KERNEL); - if (!pdata->outports) - return -ENOMEM; - - /* Children connected to this component via @outports */ - pdata->child_names = devm_kcalloc(dev, - pdata->nr_outport, - sizeof(*pdata->child_names), - GFP_KERNEL); - if (!pdata->child_names) - return -ENOMEM; - - /* Port number on the child this component is connected to */ - pdata->child_ports = devm_kcalloc(dev, - pdata->nr_outport, - sizeof(*pdata->child_ports), - GFP_KERNEL); - if (!pdata->child_ports) - return -ENOMEM; + if (pdata->nr_outport) { + pdata->conns = devm_kzalloc(dev, pdata->nr_outport * + sizeof(*pdata->conns), + GFP_KERNEL); + if (!pdata->conns) + return -ENOMEM; + } return 0; } @@ -114,17 +164,78 @@ int of_coresight_get_cpu(const struct device_node *node) } EXPORT_SYMBOL_GPL(of_coresight_get_cpu); +/* + * of_coresight_parse_endpoint : Parse the given output endpoint @ep + * and fill the connection information in @conn + * + * Parses the local port, remote device name and the remote port. + * + * Returns : + * 1 - If the parsing is successful and a connection record + * was created for an output connection. + * 0 - If the parsing completed without any fatal errors. + * -Errno - Fatal error, abort the scanning. + */ +static int of_coresight_parse_endpoint(struct device *dev, + struct device_node *ep, + struct coresight_connection *conn) +{ + int ret = 0; + struct of_endpoint endpoint, rendpoint; + struct device_node *rparent = NULL; + struct device_node *rep = NULL; + struct device *rdev = NULL; + + do { + /* Parse the local port details */ + if (of_graph_parse_endpoint(ep, &endpoint)) + break; + /* + * Get a handle on the remote endpoint and the device it is + * attached to. + */ + rep = of_graph_get_remote_endpoint(ep); + if (!rep) + break; + rparent = of_coresight_get_port_parent(rep); + if (!rparent) + break; + if (of_graph_parse_endpoint(rep, &rendpoint)) + break; + + /* If the remote device is not available, defer probing */ + rdev = of_coresight_get_endpoint_device(rparent); + if (!rdev) { + ret = -EPROBE_DEFER; + break; + } + + conn->outport = endpoint.port; + conn->child_name = devm_kstrdup(dev, + dev_name(rdev), + GFP_KERNEL); + conn->child_port = rendpoint.port; + /* Connection record updated */ + ret = 1; + } while (0); + + of_node_put(rparent); + of_node_put(rep); + put_device(rdev); + + return ret; +} + struct coresight_platform_data * of_get_coresight_platform_data(struct device *dev, const struct device_node *node) { - int i = 0, ret = 0; + int ret = 0; struct coresight_platform_data *pdata; - struct of_endpoint endpoint, rendpoint; - struct device *rdev; + struct coresight_connection *conn; struct device_node *ep = NULL; - struct device_node *rparent = NULL; - struct device_node *rport = NULL; + const struct device_node *parent = NULL; + bool legacy_binding = false; pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) @@ -132,63 +243,54 @@ of_get_coresight_platform_data(struct device *dev, /* Use device name as sysfs handle */ pdata->name = dev_name(dev); + pdata->cpu = of_coresight_get_cpu(node); /* Get the number of input and output port for this component */ of_coresight_get_ports(node, &pdata->nr_inport, &pdata->nr_outport); - if (pdata->nr_outport) { - ret = of_coresight_alloc_memory(dev, pdata); - if (ret) - return ERR_PTR(ret); - - /* Iterate through each port to discover topology */ - do { - /* Get a handle on a port */ - ep = of_graph_get_next_endpoint(node, ep); - if (!ep) - break; - - /* - * No need to deal with input ports, processing for as - * processing for output ports will deal with them. - */ - if (of_find_property(ep, "slave-mode", NULL)) - continue; - - /* Get a handle on the local endpoint */ - ret = of_graph_parse_endpoint(ep, &endpoint); - - if (ret) - continue; - - /* The local out port number */ - pdata->outports[i] = endpoint.port; - - /* - * Get a handle on the remote port and parent - * attached to it. - */ - rparent = of_graph_get_remote_port_parent(ep); - rport = of_graph_get_remote_port(ep); - - if (!rparent || !rport) - continue; + /* If there are no output connections, we are done */ + if (!pdata->nr_outport) + return pdata; - if (of_graph_parse_endpoint(rport, &rendpoint)) - continue; + ret = of_coresight_alloc_memory(dev, pdata); + if (ret) + return ERR_PTR(ret); - rdev = of_coresight_get_endpoint_device(rparent); - if (!rdev) - return ERR_PTR(-EPROBE_DEFER); - - pdata->child_names[i] = dev_name(rdev); - pdata->child_ports[i] = rendpoint.id; - - i++; - } while (ep); + parent = of_coresight_get_output_ports_node(node); + /* + * If the DT uses obsoleted bindings, the ports are listed + * under the device and we need to filter out the input + * ports. + */ + if (!parent) { + legacy_binding = true; + parent = node; + dev_warn_once(dev, "Uses obsolete Coresight DT bindings\n"); } - pdata->cpu = of_coresight_get_cpu(node); + conn = pdata->conns; + + /* Iterate through each output port to discover topology */ + while ((ep = of_graph_get_next_endpoint(parent, ep))) { + /* + * Legacy binding mixes input/output ports under the + * same parent. So, skip the input ports if we are dealing + * with legacy binding, as they processed with their + * connected output ports. + */ + if (legacy_binding && of_coresight_legacy_ep_is_input(ep)) + continue; + + ret = of_coresight_parse_endpoint(dev, ep, conn); + switch (ret) { + case 1: + conn++; /* Fall through */ + case 0: + break; + default: + return ERR_PTR(ret); + } + } return pdata; } |