diff options
Diffstat (limited to 'drivers/hwtracing/coresight/coresight-tmc-etf.c')
| -rw-r--r-- | drivers/hwtracing/coresight/coresight-tmc-etf.c | 594 |
1 files changed, 401 insertions, 193 deletions
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index e3b9fb82eb8d..8882b1c4cdc0 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -1,67 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright(C) 2016 Linaro Limited. All rights reserved. * Author: Mathieu Poirier <mathieu.poirier@linaro.org> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published by - * the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <linux/atomic.h> #include <linux/circ_buf.h> #include <linux/coresight.h> #include <linux/perf_event.h> #include <linux/slab.h> #include "coresight-priv.h" #include "coresight-tmc.h" +#include "coresight-etm-perf.h" -static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +static int tmc_set_etf_buffer(struct coresight_device *csdev, + struct perf_output_handle *handle); + +static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) { + int rc = 0; + u32 ffcr; + CS_UNLOCK(drvdata->base); /* Wait for TMCSReady bit to be set */ - tmc_wait_for_tmcready(drvdata); + rc = tmc_wait_for_tmcready(drvdata); + if (rc) { + dev_err(&drvdata->csdev->dev, + "Failed to enable: TMC not ready\n"); + CS_LOCK(drvdata->base); + return rc; + } writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); - writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | - TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT | - TMC_FFCR_TRIGON_TRIGIN, - drvdata->base + TMC_FFCR); + + ffcr = TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | TMC_FFCR_FON_FLIN | + TMC_FFCR_FON_TRIG_EVT | TMC_FFCR_TRIGON_TRIGIN; + if (drvdata->stop_on_flush) + ffcr |= TMC_FFCR_STOP_ON_FLUSH; + writel_relaxed(ffcr, drvdata->base + TMC_FFCR); writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); tmc_enable_hw(drvdata); CS_LOCK(drvdata->base); + return rc; +} + +static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = coresight_claim_device(drvdata->csdev); + + if (rc) + return rc; + + rc = __tmc_etb_enable_hw(drvdata); + if (rc) + coresight_disclaim_device(drvdata->csdev); + return rc; } static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata) { char *bufp; - u32 read_data; - int i; + u32 read_data, lost; + /* Check if the buffer wrapped around. */ + lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL; bufp = drvdata->buf; drvdata->len = 0; while (1) { - for (i = 0; i < drvdata->memwidth; i++) { - read_data = readl_relaxed(drvdata->base + TMC_RRD); - if (read_data == 0xFFFFFFFF) - return; - memcpy(bufp, &read_data, 4); - bufp += 4; - drvdata->len += 4; - } + read_data = readl_relaxed(drvdata->base + TMC_RRD); + if (read_data == 0xFFFFFFFF) + break; + memcpy(bufp, &read_data, 4); + bufp += 4; + drvdata->len += 4; } + + if (lost) + coresight_insert_barrier_packet(drvdata->buf); + return; } -static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) +static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) { CS_UNLOCK(drvdata->base); @@ -70,19 +92,33 @@ static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) * When operating in sysFS mode the content of the buffer needs to be * read before the TMC is disabled. */ - if (drvdata->mode == CS_MODE_SYSFS) + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) tmc_etb_dump_hw(drvdata); tmc_disable_hw(drvdata); CS_LOCK(drvdata->base); } -static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) { + __tmc_etb_disable_hw(drvdata); + coresight_disclaim_device(drvdata->csdev); +} + +static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = 0; + CS_UNLOCK(drvdata->base); /* Wait for TMCSReady bit to be set */ - tmc_wait_for_tmcready(drvdata); + rc = tmc_wait_for_tmcready(drvdata); + if (rc) { + dev_err(&drvdata->csdev->dev, + "Failed to enable : TMC is not ready\n"); + CS_LOCK(drvdata->base); + return rc; + } writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE); writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI, @@ -91,18 +127,52 @@ static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata) tmc_enable_hw(drvdata); CS_LOCK(drvdata->base); + return rc; +} + +static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = coresight_claim_device(drvdata->csdev); + + if (rc) + return rc; + + rc = __tmc_etf_enable_hw(drvdata); + if (rc) + coresight_disclaim_device(drvdata->csdev); + return rc; } static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + CS_UNLOCK(drvdata->base); tmc_flush_and_stop(drvdata); tmc_disable_hw(drvdata); - + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } +/* + * Return the available trace data in the buffer from @pos, with + * a maximum limit of @len, updating the @bufpp on where to + * find it. + */ +ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata, + loff_t pos, size_t len, char **bufpp) +{ + ssize_t actual = len; + + /* Adjust the len to available size @pos */ + if (pos + actual > drvdata->len) + actual = drvdata->len - pos; + if (actual > 0) + *bufpp = drvdata->buf + pos; + return actual; +} + static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) { int ret = 0; @@ -115,9 +185,9 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) * If we don't have a buffer release the lock and allocate memory. * Otherwise keep the lock and move along. */ - spin_lock_irqsave(&drvdata->spinlock, flags); + raw_spin_lock_irqsave(&drvdata->spinlock, flags); if (!drvdata->buf) { - spin_unlock_irqrestore(&drvdata->spinlock, flags); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); /* Allocating the memory here while outside of the spinlock */ buf = kzalloc(drvdata->size, GFP_KERNEL); @@ -125,7 +195,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) return -ENOMEM; /* Let's try again */ - spin_lock_irqsave(&drvdata->spinlock, flags); + raw_spin_lock_irqsave(&drvdata->spinlock, flags); } if (drvdata->reading) { @@ -138,8 +208,10 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) * sink is already enabled no memory is needed and the HW need not be * touched. */ - if (drvdata->mode == CS_MODE_SYSFS) + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { + csdev->refcnt++; goto out; + } /* * If drvdata::buf isn't NULL, memory was allocated for a previous @@ -156,11 +228,16 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) used = true; drvdata->buf = buf; } - - drvdata->mode = CS_MODE_SYSFS; - tmc_etb_enable_hw(drvdata); + ret = tmc_etb_enable_hw(drvdata); + if (!ret) { + coresight_set_mode(csdev, CS_MODE_SYSFS); + csdev->refcnt++; + } else { + /* Free up the buffer if we failed to enable */ + used = false; + } out: - spin_unlock_irqrestore(&drvdata->spinlock, flags); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); /* Free memory outside the spinlock if need be */ if (!used) @@ -169,47 +246,76 @@ out: return ret; } -static int tmc_enable_etf_sink_perf(struct coresight_device *csdev) +static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, + struct coresight_path *path) { int ret = 0; + pid_t pid; unsigned long flags; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct perf_output_handle *handle = path->handle; + struct cs_buffers *buf = etm_perf_sink_config(handle); - spin_lock_irqsave(&drvdata->spinlock, flags); - if (drvdata->reading) { + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + do { ret = -EINVAL; - goto out; - } + if (drvdata->reading) + break; + /* + * No need to continue if the ETB/ETF is already operated + * from sysFS. + */ + if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { + ret = -EBUSY; + break; + } - /* - * In Perf mode there can be only one writer per sink. There - * is also no need to continue if the ETB/ETR is already operated - * from sysFS. - */ - if (drvdata->mode != CS_MODE_DISABLED) { - ret = -EINVAL; - goto out; - } + /* Get a handle on the pid of the process to monitor */ + pid = buf->pid; - drvdata->mode = CS_MODE_PERF; - tmc_etb_enable_hw(drvdata); -out: - spin_unlock_irqrestore(&drvdata->spinlock, flags); + if (drvdata->pid != -1 && drvdata->pid != pid) { + ret = -EBUSY; + break; + } + + ret = tmc_set_etf_buffer(csdev, handle); + if (ret) + break; + + /* + * No HW configuration is needed if the sink is already in + * use for this session. + */ + if (drvdata->pid == pid) { + csdev->refcnt++; + break; + } + + ret = tmc_etb_enable_hw(drvdata); + if (!ret) { + /* Associate with monitored process. */ + drvdata->pid = pid; + coresight_set_mode(csdev, CS_MODE_PERF); + csdev->refcnt++; + } + } while (0); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); return ret; } -static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode) +static int tmc_enable_etf_sink(struct coresight_device *csdev, + enum cs_mode mode, + struct coresight_path *path) { int ret; - struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); switch (mode) { case CS_MODE_SYSFS: ret = tmc_enable_etf_sink_sysfs(csdev); break; case CS_MODE_PERF: - ret = tmc_enable_etf_sink_perf(csdev); + ret = tmc_enable_etf_sink_perf(csdev, path); break; /* We shouldn't be here */ default: @@ -220,86 +326,113 @@ static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode) if (ret) return ret; - dev_info(drvdata->dev, "TMC-ETB/ETF enabled\n"); + dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n"); return 0; } -static void tmc_disable_etf_sink(struct coresight_device *csdev) +static int tmc_disable_etf_sink(struct coresight_device *csdev) { unsigned long flags; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - spin_lock_irqsave(&drvdata->spinlock, flags); + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + if (drvdata->reading) { - spin_unlock_irqrestore(&drvdata->spinlock, flags); - return; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return -EBUSY; } - /* Disable the TMC only if it needs to */ - if (drvdata->mode != CS_MODE_DISABLED) { - tmc_etb_disable_hw(drvdata); - drvdata->mode = CS_MODE_DISABLED; + csdev->refcnt--; + if (csdev->refcnt) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return -EBUSY; } - spin_unlock_irqrestore(&drvdata->spinlock, flags); + /* Complain if we (somehow) got out of sync */ + WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED); + tmc_etb_disable_hw(drvdata); + /* Dissociate from monitored process. */ + drvdata->pid = -1; + coresight_set_mode(csdev, CS_MODE_DISABLED); - dev_info(drvdata->dev, "TMC-ETB/ETF disabled\n"); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n"); + return 0; } static int tmc_enable_etf_link(struct coresight_device *csdev, - int inport, int outport) + struct coresight_connection *in, + struct coresight_connection *out) { + int ret = 0; unsigned long flags; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + bool first_enable = false; - spin_lock_irqsave(&drvdata->spinlock, flags); + raw_spin_lock_irqsave(&drvdata->spinlock, flags); if (drvdata->reading) { - spin_unlock_irqrestore(&drvdata->spinlock, flags); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); return -EBUSY; } - tmc_etf_enable_hw(drvdata); - drvdata->mode = CS_MODE_SYSFS; - spin_unlock_irqrestore(&drvdata->spinlock, flags); + if (csdev->refcnt == 0) { + ret = tmc_etf_enable_hw(drvdata); + if (!ret) { + coresight_set_mode(csdev, CS_MODE_SYSFS); + first_enable = true; + } + } + if (!ret) + csdev->refcnt++; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "TMC-ETF enabled\n"); - return 0; + if (first_enable) + dev_dbg(&csdev->dev, "TMC-ETF enabled\n"); + return ret; } static void tmc_disable_etf_link(struct coresight_device *csdev, - int inport, int outport) + struct coresight_connection *in, + struct coresight_connection *out) { unsigned long flags; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + bool last_disable = false; - spin_lock_irqsave(&drvdata->spinlock, flags); + raw_spin_lock_irqsave(&drvdata->spinlock, flags); if (drvdata->reading) { - spin_unlock_irqrestore(&drvdata->spinlock, flags); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); return; } - tmc_etf_disable_hw(drvdata); - drvdata->mode = CS_MODE_DISABLED; - spin_unlock_irqrestore(&drvdata->spinlock, flags); + csdev->refcnt--; + if (csdev->refcnt == 0) { + tmc_etf_disable_hw(drvdata); + coresight_set_mode(csdev, CS_MODE_DISABLED); + last_disable = true; + } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); - dev_info(drvdata->dev, "TMC-ETF disabled\n"); + if (last_disable) + dev_dbg(&csdev->dev, "TMC-ETF disabled\n"); } -static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, int cpu, - void **pages, int nr_pages, bool overwrite) +static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, + struct perf_event *event, void **pages, + int nr_pages, bool overwrite) { int node; struct cs_buffers *buf; - if (cpu == -1) - cpu = smp_processor_id(); - node = cpu_to_node(cpu); + node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); /* Allocate memory structure for interaction with Perf */ buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node); if (!buf) return NULL; + buf->pid = task_pid_nr(event->owner); buf->snapshot = overwrite; buf->nr_pages = nr_pages; buf->data_pages = pages; @@ -315,15 +448,17 @@ static void tmc_free_etf_buffer(void *config) } static int tmc_set_etf_buffer(struct coresight_device *csdev, - struct perf_output_handle *handle, - void *sink_config) + struct perf_output_handle *handle) { int ret = 0; unsigned long head; - struct cs_buffers *buf = sink_config; + struct cs_buffers *buf = etm_perf_sink_config(handle); + + if (!buf) + return -EINVAL; /* wrap head around to the amount of space we have */ - head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1); + head = handle->head & (((unsigned long)buf->nr_pages << PAGE_SHIFT) - 1); /* find the page to write to */ buf->cur = head / PAGE_SIZE; @@ -336,60 +471,40 @@ static int tmc_set_etf_buffer(struct coresight_device *csdev, return ret; } -static unsigned long tmc_reset_etf_buffer(struct coresight_device *csdev, - struct perf_output_handle *handle, - void *sink_config) -{ - long size = 0; - struct cs_buffers *buf = sink_config; - - if (buf) { - /* - * In snapshot mode ->data_size holds the new address of the - * ring buffer's head. The size itself is the whole address - * range since we want the latest information. - */ - if (buf->snapshot) - handle->head = local_xchg(&buf->data_size, - buf->nr_pages << PAGE_SHIFT); - /* - * Tell the tracer PMU how much we got in this run and if - * something went wrong along the way. Nobody else can use - * this cs_buffers instance until we are done. As such - * resetting parameters here and squaring off with the ring - * buffer API in the tracer PMU is fine. - */ - size = local_xchg(&buf->data_size, 0); - } - - return size; -} - -static void tmc_update_etf_buffer(struct coresight_device *csdev, +static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, struct perf_output_handle *handle, void *sink_config) { + bool lost = false; int i, cur; + const u32 *barrier; u32 *buf_ptr; - u32 read_ptr, write_ptr; - u32 status, to_read; - unsigned long offset; + u64 read_ptr, write_ptr; + u32 status; + unsigned long offset, to_read = 0, flags; struct cs_buffers *buf = sink_config; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct perf_event *event = handle->event; if (!buf) - return; + return 0; /* This shouldn't happen */ - if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF)) - return; + if (WARN_ON_ONCE(coresight_get_mode(csdev) != CS_MODE_PERF)) + return 0; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + /* Don't do anything if another tracer is using this sink */ + if (csdev->refcnt != 1) + goto out; CS_UNLOCK(drvdata->base); tmc_flush_and_stop(drvdata); - read_ptr = readl_relaxed(drvdata->base + TMC_RRP); - write_ptr = readl_relaxed(drvdata->base + TMC_RWP); + read_ptr = tmc_read_rrp(drvdata); + write_ptr = tmc_read_rwp(drvdata); /* * Get a hold of the status register and see if a wrap around @@ -397,7 +512,7 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev, */ status = readl_relaxed(drvdata->base + TMC_STS); if (status & TMC_STS_FULL) { - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + lost = true; to_read = drvdata->size; } else { to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size); @@ -406,33 +521,16 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev, /* * The TMC RAM buffer may be bigger than the space available in the * perf ring buffer (handle->size). If so advance the RRP so that we - * get the latest trace data. + * get the latest trace data. In snapshot mode none of that matters + * since we are expected to clobber stale data in favour of the latest + * traces. */ - if (to_read > handle->size) { - u32 mask = 0; - - /* - * The value written to RRP must be byte-address aligned to - * the width of the trace memory databus _and_ to a frame - * boundary (16 byte), whichever is the biggest. For example, - * for 32-bit, 64-bit and 128-bit wide trace memory, the four - * LSBs must be 0s. For 256-bit wide trace memory, the five - * LSBs must be 0s. - */ - switch (drvdata->memwidth) { - case TMC_MEM_INTF_WIDTH_32BITS: - case TMC_MEM_INTF_WIDTH_64BITS: - case TMC_MEM_INTF_WIDTH_128BITS: - mask = GENMASK(31, 5); - break; - case TMC_MEM_INTF_WIDTH_256BITS: - mask = GENMASK(31, 6); - break; - } + if (!buf->snapshot && to_read > handle->size) { + u32 mask = tmc_get_memwidth_mask(drvdata); /* * Make sure the new size is aligned in accordance with the - * requirement explained above. + * requirement explained in function tmc_get_memwidth_mask(). */ to_read = handle->size & mask; /* Move the RAM read pointer up */ @@ -441,18 +539,33 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev, if (read_ptr > (drvdata->size - 1)) read_ptr -= drvdata->size; /* Tell the HW */ - writel_relaxed(read_ptr, drvdata->base + TMC_RRP); - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + tmc_write_rrp(drvdata, read_ptr); + lost = true; } + /* + * Don't set the TRUNCATED flag in snapshot mode because 1) the + * captured buffer is expected to be truncated and 2) a full buffer + * prevents the event from being re-enabled by the perf core, + * resulting in stale data being send to user space. + */ + if (!buf->snapshot && lost) + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + cur = buf->cur; offset = buf->offset; + barrier = coresight_barrier_pkt; /* for every byte to read */ for (i = 0; i < to_read; i += 4) { buf_ptr = buf->data_pages[cur] + offset; *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD); + if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) { + *buf_ptr = *barrier; + barrier++; + } + offset += 4; if (offset >= PAGE_SIZE) { offset = 0; @@ -463,17 +576,106 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev, } /* - * In snapshot mode all we have to do is communicate to - * perf_aux_output_end() the address of the current head. In full - * trace mode the same function expects a size to move rb->aux_head - * forward. + * In snapshot mode we simply increment the head by the number of byte + * that were written. User space will figure out how many bytes to get + * from the AUX buffer based on the position of the head. */ if (buf->snapshot) - local_set(&buf->data_size, (cur * PAGE_SIZE) + offset); - else - local_add(to_read, &buf->data_size); + handle->head += to_read; + /* + * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace + * data before the aux_head is updated via perf_aux_output_end(), which + * is expected by the perf ring buffer. + */ CS_LOCK(drvdata->base); + + /* + * If the event is active, it is triggered during an AUX pause. + * Re-enable the sink so that it is ready when AUX resume is invoked. + */ + if (!event->hw.state) + __tmc_etb_enable_hw(drvdata); + +out: + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + return to_read; +} + +static int tmc_panic_sync_etf(struct coresight_device *csdev) +{ + u32 val; + struct tmc_crash_metadata *mdata; + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + mdata = (struct tmc_crash_metadata *)drvdata->crash_mdata.vaddr; + + /* Make sure we have valid reserved memory */ + if (!tmc_has_reserved_buffer(drvdata) || + !tmc_has_crash_mdata_buffer(drvdata)) + return 0; + + tmc_crashdata_set_invalid(drvdata); + + CS_UNLOCK(drvdata->base); + + /* Proceed only if ETF is enabled or configured as sink */ + val = readl(drvdata->base + TMC_CTL); + if (!(val & TMC_CTL_CAPT_EN)) + goto out; + val = readl(drvdata->base + TMC_MODE); + if (val != TMC_MODE_CIRCULAR_BUFFER) + goto out; + + val = readl(drvdata->base + TMC_FFSR); + /* Do manual flush and stop only if its not auto-stopped */ + if (!(val & TMC_FFSR_FT_STOPPED)) { + dev_dbg(&csdev->dev, + "%s: Triggering manual flush\n", __func__); + tmc_flush_and_stop(drvdata); + } else + tmc_wait_for_tmcready(drvdata); + + /* Sync registers from hardware to metadata region */ + mdata->tmc_sts = readl(drvdata->base + TMC_STS); + mdata->tmc_mode = readl(drvdata->base + TMC_MODE); + mdata->tmc_ffcr = readl(drvdata->base + TMC_FFCR); + mdata->tmc_ffsr = readl(drvdata->base + TMC_FFSR); + + /* Sync Internal SRAM to reserved trace buffer region */ + drvdata->buf = drvdata->resrv_buf.vaddr; + tmc_etb_dump_hw(drvdata); + /* Store as per RSZ register convention */ + mdata->tmc_ram_size = drvdata->len >> 2; + + /* Other fields for processing trace buffer reads */ + mdata->tmc_rrp = 0; + mdata->tmc_dba = 0; + mdata->tmc_rwp = drvdata->len; + mdata->trace_paddr = drvdata->resrv_buf.paddr; + + mdata->version = CS_CRASHDATA_VERSION; + + /* + * Make sure all previous writes are ordered, + * before we mark valid + */ + dmb(sy); + mdata->valid = true; + /* + * Below order need to maintained, since crc of metadata + * is dependent on first + */ + mdata->crc32_tdata = find_crash_tracedata_crc(drvdata, mdata); + mdata->crc32_mdata = find_crash_metadata_crc(mdata); + + tmc_disable_hw(drvdata); + + dev_dbg(&csdev->dev, "%s: success\n", __func__); +out: + CS_UNLOCK(drvdata->base); + return 0; } static const struct coresight_ops_sink tmc_etf_sink_ops = { @@ -481,8 +683,6 @@ static const struct coresight_ops_sink tmc_etf_sink_ops = { .disable = tmc_disable_etf_sink, .alloc_buffer = tmc_alloc_etf_buffer, .free_buffer = tmc_free_etf_buffer, - .set_buffer = tmc_set_etf_buffer, - .reset_buffer = tmc_reset_etf_buffer, .update_buffer = tmc_update_etf_buffer, }; @@ -491,6 +691,10 @@ static const struct coresight_ops_link tmc_etf_link_ops = { .disable = tmc_disable_etf_link, }; +static const struct coresight_ops_panic tmc_etf_sync_ops = { + .sync = tmc_panic_sync_etf, +}; + const struct coresight_ops tmc_etb_cs_ops = { .sink_ops = &tmc_etf_sink_ops, }; @@ -498,6 +702,7 @@ const struct coresight_ops tmc_etb_cs_ops = { const struct coresight_ops tmc_etf_cs_ops = { .sink_ops = &tmc_etf_sink_ops, .link_ops = &tmc_etf_link_ops, + .panic_ops = &tmc_etf_sync_ops, }; int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) @@ -511,22 +716,15 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) drvdata->config_type != TMC_CONFIG_TYPE_ETF)) return -EINVAL; - spin_lock_irqsave(&drvdata->spinlock, flags); + raw_spin_lock_irqsave(&drvdata->spinlock, flags); if (drvdata->reading) { ret = -EBUSY; goto out; } - /* There is no point in reading a TMC in HW FIFO mode */ - mode = readl_relaxed(drvdata->base + TMC_MODE); - if (mode != TMC_MODE_CIRCULAR_BUFFER) { - ret = -EINVAL; - goto out; - } - /* Don't interfere if operated from Perf */ - if (drvdata->mode == CS_MODE_PERF) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_PERF) { ret = -EINVAL; goto out; } @@ -538,12 +736,19 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) } /* Disable the TMC if need be */ - if (drvdata->mode == CS_MODE_SYSFS) - tmc_etb_disable_hw(drvdata); + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { + /* There is no point in reading a TMC in HW FIFO mode */ + mode = readl_relaxed(drvdata->base + TMC_MODE); + if (mode != TMC_MODE_CIRCULAR_BUFFER) { + ret = -EINVAL; + goto out; + } + __tmc_etb_disable_hw(drvdata); + } drvdata->reading = true; out: - spin_unlock_irqrestore(&drvdata->spinlock, flags); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); return ret; } @@ -559,17 +764,16 @@ int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) drvdata->config_type != TMC_CONFIG_TYPE_ETF)) return -EINVAL; - spin_lock_irqsave(&drvdata->spinlock, flags); - - /* There is no point in reading a TMC in HW FIFO mode */ - mode = readl_relaxed(drvdata->base + TMC_MODE); - if (mode != TMC_MODE_CIRCULAR_BUFFER) { - spin_unlock_irqrestore(&drvdata->spinlock, flags); - return -EINVAL; - } + raw_spin_lock_irqsave(&drvdata->spinlock, flags); /* Re-enable the TMC if need be */ - if (drvdata->mode == CS_MODE_SYSFS) { + if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { + /* There is no point in reading a TMC in HW FIFO mode */ + mode = readl_relaxed(drvdata->base + TMC_MODE); + if (mode != TMC_MODE_CIRCULAR_BUFFER) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return -EINVAL; + } /* * The trace run will continue with the same allocated trace * buffer. As such zero-out the buffer so that we don't end @@ -579,7 +783,11 @@ int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) * can't be NULL. */ memset(drvdata->buf, 0, drvdata->size); - tmc_etb_enable_hw(drvdata); + /* + * Ignore failures to enable the TMC to make sure, we don't + * leave the TMC in a "reading" state. + */ + __tmc_etb_enable_hw(drvdata); } else { /* * The ETB/ETF is not tracing and the buffer was just read. @@ -590,7 +798,7 @@ int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) } drvdata->reading = false; - spin_unlock_irqrestore(&drvdata->spinlock, flags); + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); /* * Free allocated memory outside of the spinlock. There is no need |