// SPDX-License-Identifier: GPL-2.0 /* * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight * sink device could then pair with an appropriate per-cpu coresight source * device (ETE) thus generating required trace data. Trace can be enabled * via the perf framework. * * The AUX buffer handling is inspired from Arm SPE PMU driver. * * Copyright (C) 2020 ARM Ltd. * * Author: Anshuman Khandual */ #define DRVNAME "arm_trbe" #define pr_fmt(fmt) DRVNAME ": " fmt #include #include "coresight-trbe.h" #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) /* * A padding packet that will help the user space tools * in skipping relevant sections in the captured trace * data which could not be decoded. TRBE doesn't support * formatting the trace data, unlike the legacy CoreSight * sinks and thus we use ETE trace packets to pad the * sections of the buffer. */ #define ETE_IGNORE_PACKET 0x70 /* * Minimum amount of meaningful trace will contain: * A-Sync, Trace Info, Trace On, Address, Atom. * This is about 44bytes of ETE trace. To be on * the safer side, we assume 64bytes is the minimum * space required for a meaningful session, before * we hit a "WRAP" event. */ #define TRBE_TRACE_MIN_BUF_SIZE 64 enum trbe_fault_action { TRBE_FAULT_ACT_WRAP, TRBE_FAULT_ACT_SPURIOUS, TRBE_FAULT_ACT_FATAL, }; struct trbe_buf { /* * Even though trbe_base represents vmap() * mapped allocated buffer's start address, * it's being as unsigned long for various * arithmetic and comparision operations & * also to be consistent with trbe_write & * trbe_limit sibling pointers. */ unsigned long trbe_base; unsigned long trbe_limit; unsigned long trbe_write; int nr_pages; void **pages; bool snapshot; struct trbe_cpudata *cpudata; }; struct trbe_cpudata { bool trbe_flag; u64 trbe_align; int cpu; enum cs_mode mode; struct trbe_buf *buf; struct trbe_drvdata *drvdata; }; struct trbe_drvdata { struct trbe_cpudata __percpu *cpudata; struct perf_output_handle * __percpu *handle; struct hlist_node hotplug_node; int irq; cpumask_t supported_cpus; enum cpuhp_state trbe_online; struct platform_device *pdev; }; static int trbe_alloc_node(struct perf_event *event) { if (event->cpu == -1) return NUMA_NO_NODE; return cpu_to_node(event->cpu); } static void trbe_drain_buffer(void) { tsb_csync(); dsb(nsh); } static void trbe_drain_and_disable_local(void) { u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); trbe_drain_buffer(); /* * Disable the TRBE without clearing LIMITPTR which * might be required for fetching the buffer limits. */ trblimitr &= ~TRBLIMITR_ENABLE; write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1); isb(); } static void trbe_reset_local(void) { trbe_drain_and_disable_local(); write_sysreg_s(0, SYS_TRBLIMITR_EL1); write_sysreg_s(0, SYS_TRBPTR_EL1); write_sysreg_s(0, SYS_TRBBASER_EL1); write_sysreg_s(0, SYS_TRBSR_EL1); } static void trbe_stop_and_truncate_event(struct perf_output_handle *handle) { struct trbe_buf *buf = etm_perf_sink_config(handle); /* * We cannot proceed with the buffer collection and we * do not have any data for the current session. The * etm_perf driver expects to close out the aux_buffer * at event_stop(). So disable the TRBE here and leave * the update_buffer() to return a 0 size. */ trbe_drain_and_disable_local(); perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; } /* * TRBE Buffer Management * * The TRBE buffer spans from the base pointer till the limit pointer. When enabled, * it starts writing trace data from the write pointer onward till the limit pointer. * When the write pointer reaches the address just before the limit pointer, it gets * wrapped around again to the base pointer. This is called a TRBE wrap event, which * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and * LIMIT pointers. * * Wrap around with an IRQ * ------ < ------ < ------- < ----- < ----- * | | * ------ > ------ > ------- > ----- > ----- * * +---------------+-----------------------+ * | | | * +---------------+-----------------------+ * Base Pointer Write Pointer Limit Pointer * * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write * pointer can be aligned to the implementation defined TRBE trace buffer alignment * as captured in trbe_cpudata->trbe_align. * * * head tail wakeup * +---------------------------------------+----- ~ ~ ------ * |$$$$$$$|################|$$$$$$$$$$$$$$| | * +---------------------------------------+----- ~ ~ ------ * Base Pointer Write Pointer Limit Pointer * * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing * values which tracks all the driver writes and user reads from the perf auxiliary * buffer. Generally [head..tail] is the area where the driver can write into unless * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and * configured depending on the perf_output_handle indices, so that the driver does * not override into areas in the perf auxiliary buffer which is being or yet to be * consumed from the user space. The enabled TRBE buffer area is a moving subset of * the allocated perf auxiliary buffer. */ static void trbe_pad_buf(struct perf_output_handle *handle, int len) { struct trbe_buf *buf = etm_perf_sink_config(handle); u64 head = PERF_IDX2OFF(handle->head, buf); memset((void *)buf->trbe_base + head, ETE_IGNORE_PACKET, len); if (!buf->snapshot) perf_aux_output_skip(handle, len); } static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle) { struct trbe_buf *buf = etm_perf_sink_config(handle); /* * The ETE trace has alignment synchronization packets allowing * the decoder to reset in case of an overflow or corruption. * So we can use the entire buffer for the snapshot mode. */ return buf->nr_pages * PAGE_SIZE; } /* * TRBE Limit Calculation * * The following markers are used to illustrate various TRBE buffer situations. * * $$$$ - Data area, unconsumed captured trace data, not to be overridden * #### - Free area, enabled, trace will be written * %%%% - Free area, disabled, trace will not be written * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped */ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle) { struct trbe_buf *buf = etm_perf_sink_config(handle); struct trbe_cpudata *cpudata = buf->cpudata; const u64 bufsize = buf->nr_pages * PAGE_SIZE; u64 limit = bufsize; u64 head, tail, wakeup; head = PERF_IDX2OFF(handle->head, buf); /* * head * ------->| * | * head TRBE align tail * +----|-------|---------------|-------+ * |$$$$|=======|###############|$$$$$$$| * +----|-------|---------------|-------+ * trbe_base trbe_base + nr_pages * * Perf aux buffer output head position can be misaligned depending on * various factors including user space reads. In case misaligned, head * needs to be aligned before TRBE can be configured. Pad the alignment * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools * and skip this section thus advancing the head. */ if (!IS_ALIGNED(head, cpudata->trbe_align)) { unsigned long delta = roundup(head, cpudata->trbe_align) - head; delta = min(delta, handle->size); trbe_pad_buf(handle, delta); head = PERF_IDX2OFF(handle->head, buf); } /* * head = tail (size = 0) * +----|-------------------------------+ * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ | * +----|-------------------------------+ * trbe_base trbe_base + nr_pages * * Perf aux buffer does not have any space for the driver to write into. * Just communicate trace truncation event to the user space by marking * it with PERF_AUX_FLAG_TRUNCATED. */ if (!handle->size) { perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); return 0; } /* Compute the tail and wakeup indices now that we've aligned head */ tail = PERF_IDX2OFF(handle->head + handle->size, buf); wakeup = PERF_IDX2OFF(handle->wakeup, buf); /* * Lets calculate the buffer area which TRBE could write into. There * are three possible scenarios here. Limit needs to be aligned with * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the * unconsumed data. * * 1) head < tail * * head tail * +----|-----------------------|-------+ * |$$$$|#######################|$$$$$$$| * +----|-----------------------|-------+ * trbe_base limit trbe_base + nr_pages * * TRBE could write into [head..tail] area. Unless the tail is right at * the end of the buffer, neither an wrap around nor an IRQ is expected * while being enabled. * * 2) head == tail * * head = tail (size > 0) * +----|-------------------------------+ * |%%%%|###############################| * +----|-------------------------------+ * trbe_base limit = trbe_base + nr_pages * * TRBE should just write into [head..base + nr_pages] area even though * the entire buffer is empty. Reason being, when the trace reaches the * end of the buffer, it will just wrap around with an IRQ giving an * opportunity to reconfigure the buffer. * * 3) tail < head * * tail head * +----|-----------------------|-------+ * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######| * +----|-----------------------|-------+ * trbe_base limit = trbe_base + nr_pages * * TRBE should just write into [head..base + nr_pages] area even though * the [trbe_base..tail] is also empty. Reason being, when the trace * reaches the end of the buffer, it will just wrap around with an IRQ * giving an opportunity to reconfigure the buffer. */ if (head < tail) limit = round_down(tail, PAGE_SIZE); /* * Wakeup may be arbitrarily far into the future. If it's not in the * current generation, either we'll wrap before hitting it, or it's * in the past and has been handled already. * * If there's a wakeup before we wrap, arrange to be woken up by the * page boundary following it. Keep the tail boundary if that's lower. * * head wakeup tail * +----|---------------|-------|-------+ * |$$$$|###############|%%%%%%%|$$$$$$$| * +----|---------------|-------|-------+ * trbe_base limit trbe_base + nr_pages */ if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) limit = min(limit, round_up(wakeup, PAGE_SIZE)); /* * There are two situation when this can happen i.e limit is before * the head and hence TRBE cannot be configured. * * 1) head < tail (aligned down with PAGE_SIZE) and also they are both * within the same PAGE size range. * * PAGE_SIZE * |----------------------| * * limit head tail * +------------|------|--------|-------+ * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$| * +------------|------|--------|-------+ * trbe_base trbe_base + nr_pages * * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both * head and wakeup are within same PAGE size range. * * PAGE_SIZE * |----------------------| * * limit head wakeup tail * +----|------|-------|--------|-------+ * |$$$$$$$$$$$|=======|========|$$$$$$$| * +----|------|-------|--------|-------+ * trbe_base trbe_base + nr_pages */ if (limit > head) return limit; trbe_pad_buf(handle, handle->size); perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); return 0; } static unsigned long trbe_normal_offset(struct perf_output_handle *handle) { struct trbe_buf *buf = perf_get_aux(handle); u64 limit = __trbe_normal_offset(handle); u64 head = PERF_IDX2OFF(handle->head, buf); /* * If the head is too close to the limit and we don't * have space for a meaningful run, we rather pad it * and start fresh. */ if (limit && (limit - head < TRBE_TRACE_MIN_BUF_SIZE)) { trbe_pad_buf(handle, limit - head); limit = __trbe_normal_offset(handle); } return limit; } static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle) { struct trbe_buf *buf = etm_perf_sink_config(handle); unsigned long offset; if (buf->snapshot) offset = trbe_snapshot_offset(handle); else offset = trbe_normal_offset(handle); return buf->trbe_base + offset; } static void clr_trbe_status(void) { u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1); WARN_ON(is_trbe_enabled()); trbsr &= ~TRBSR_IRQ; trbsr &= ~TRBSR_TRG; trbsr &= ~TRBSR_WRAP; trbsr &= ~(TRBSR_EC_MASK << TRBSR_EC_SHIFT); trbsr &= ~(TRBSR_BSC_MASK << TRBSR_BSC_SHIFT); trbsr &= ~TRBSR_STOP; write_sysreg_s(trbsr, SYS_TRBSR_EL1); } static void set_trbe_limit_pointer_enabled(unsigned long addr) { u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_LIMIT_SHIFT))); WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); trblimitr &= ~TRBLIMITR_NVM; trblimitr &= ~(TRBLIMITR_FILL_MODE_MASK << TRBLIMITR_FILL_MODE_SHIFT); trblimitr &= ~(TRBLIMITR_TRIG_MODE_MASK << TRBLIMITR_TRIG_MODE_SHIFT); trblimitr &= ~(TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT); /* * Fill trace buffer mode is used here while configuring the * TRBE for trace capture. In this particular mode, the trace * collection is stopped and a maintenance interrupt is raised * when the current write pointer wraps. This pause in trace * collection gives the software an opportunity to capture the * trace data in the interrupt handler, before reconfiguring * the TRBE. */ trblimitr |= (TRBE_FILL_MODE_FILL & TRBLIMITR_FILL_MODE_MASK) << TRBLIMITR_FILL_MODE_SHIFT; /* * Trigger mode is not used here while configuring the TRBE for * the trace capture. Hence just keep this in the ignore mode. */ trblimitr |= (TRBE_TRIG_MODE_IGNORE & TRBLIMITR_TRIG_MODE_MASK) << TRBLIMITR_TRIG_MODE_SHIFT; trblimitr |= (addr & PAGE_MASK); trblimitr |= TRBLIMITR_ENABLE; write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1); /* Synchronize the TRBE enable event */ isb(); } static void trbe_enable_hw(struct trbe_buf *buf) { WARN_ON(buf->trbe_write < buf->trbe_base); WARN_ON(buf->trbe_write >= buf->trbe_limit); set_trbe_disabled(); isb(); clr_trbe_status(); set_trbe_base_pointer(buf->trbe_base); set_trbe_write_pointer(buf->trbe_write); /* * Synchronize all the register updates * till now before enabling the TRBE. */ isb(); set_trbe_limit_pointer_enabled(buf->trbe_limit); } static enum trbe_fault_action trbe_get_fault_act(u64 trbsr) { int ec = get_trbe_ec(trbsr); int bsc = get_trbe_bsc(trbsr); WARN_ON(is_trbe_running(trbsr)); if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr)) return TRBE_FAULT_ACT_FATAL; if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT)) return TRBE_FAULT_ACT_FATAL; if (is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) { if (get_trbe_write_pointer() == get_trbe_base_pointer()) return TRBE_FAULT_ACT_WRAP; } return TRBE_FAULT_ACT_SPURIOUS; } static void *arm_trbe_alloc_buffer(struct coresight_device *csdev, struct perf_event *event, void **pages, int nr_pages, bool snapshot) { struct trbe_buf *buf; struct page **pglist; int i; /* * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with * just a single page, there would not be any room left while writing * into a partially filled TRBE buffer after the page size alignment. * Hence restrict the minimum buffer size as two pages. */ if (nr_pages < 2) return NULL; buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event)); if (!buf) return ERR_PTR(-ENOMEM); pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL); if (!pglist) { kfree(buf); return ERR_PTR(-ENOMEM); } for (i = 0; i < nr_pages; i++) pglist[i] = virt_to_page(pages[i]); buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); if (!buf->trbe_base) { kfree(pglist); kfree(buf); return ERR_PTR(-ENOMEM); } buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE; buf->trbe_write = buf->trbe_base; buf->snapshot = snapshot; buf->nr_pages = nr_pages; buf->pages = pages; kfree(pglist); return buf; } static void arm_trbe_free_buffer(void *config) { struct trbe_buf *buf = config; vunmap((void *)buf->trbe_base); kfree(buf); } static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, struct perf_output_handle *handle, void *config) { struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); struct trbe_buf *buf = config; enum trbe_fault_action act; unsigned long size, offset; unsigned long write, base, status; unsigned long flags; WARN_ON(buf->cpudata != cpudata); WARN_ON(cpudata->cpu != smp_processor_id()); WARN_ON(cpudata->drvdata != drvdata); if (cpudata->mode != CS_MODE_PERF) return 0; perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); /* * We are about to disable the TRBE. And this could in turn * fill up the buffer triggering, an IRQ. This could be consumed * by the PE asynchronously, causing a race here against * the IRQ handler in closing out the handle. So, let us * make sure the IRQ can't trigger while we are collecting * the buffer. We also make sure that a WRAP event is handled * accordingly. */ local_irq_save(flags); /* * If the TRBE was disabled due to lack of space in the AUX buffer or a * spurious fault, the driver leaves it disabled, truncating the buffer. * Since the etm_perf driver expects to close out the AUX buffer, the * driver skips it. Thus, just pass in 0 size here to indicate that the * buffer was truncated. */ if (!is_trbe_enabled()) { size = 0; goto done; } /* * perf handle structure needs to be shared with the TRBE IRQ handler for * capturing trace data and restarting the handle. There is a probability * of an undefined reference based crash when etm event is being stopped * while a TRBE IRQ also getting processed. This happens due the release * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping * the TRBE here will ensure that no IRQ could be generated when the perf * handle gets freed in etm_event_stop(). */ trbe_drain_and_disable_local(); write = get_trbe_write_pointer(); base = get_trbe_base_pointer(); /* Check if there is a pending interrupt and handle it here */ status = read_sysreg_s(SYS_TRBSR_EL1); if (is_trbe_irq(status)) { /* * Now that we are handling the IRQ here, clear the IRQ * from the status, to let the irq handler know that it * is taken care of. */ clr_trbe_irq(); isb(); act = trbe_get_fault_act(status); /* * If this was not due to a WRAP event, we have some * errors and as such buffer is empty. */ if (act != TRBE_FAULT_ACT_WRAP) { size = 0; goto done; } /* * Otherwise, the buffer is full and the write pointer * has reached base. Adjust this back to the Limit pointer * for correct size. Also, mark the buffer truncated. */ write = get_trbe_limit_pointer(); perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); } offset = write - base; if (WARN_ON_ONCE(offset < PERF_IDX2OFF(handle->head, buf))) size = 0; else size = offset - PERF_IDX2OFF(handle->head, buf); done: local_irq_restore(flags); if (buf->snapshot) handle->head += size; return size; } static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data) { struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); struct perf_output_handle *handle = data; struct trbe_buf *buf = etm_perf_sink_config(handle); WARN_ON(cpudata->cpu != smp_processor_id()); WARN_ON(cpudata->drvdata != drvdata); if (mode != CS_MODE_PERF) return -EINVAL; *this_cpu_ptr(drvdata->handle) = handle; cpudata->buf = buf; cpudata->mode = mode; buf->cpudata = cpudata; buf->trbe_limit = compute_trbe_buffer_limit(handle); buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); if (buf->trbe_limit == buf->trbe_base) { trbe_stop_and_truncate_event(handle); return 0; } trbe_enable_hw(buf); return 0; } static int arm_trbe_disable(struct coresight_device *csdev) { struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); struct trbe_buf *buf = cpudata->buf; WARN_ON(buf->cpudata != cpudata); WARN_ON(cpudata->cpu != smp_processor_id()); WARN_ON(cpudata->drvdata != drvdata); if (cpudata->mode != CS_MODE_PERF) return -EINVAL; trbe_drain_and_disable_local(); buf->cpudata = NULL; cpudata->buf = NULL; cpudata->mode = CS_MODE_DISABLED; return 0; } static void trbe_handle_spurious(struct perf_output_handle *handle) { struct trbe_buf *buf = etm_perf_sink_config(handle); buf->trbe_limit = compute_trbe_buffer_limit(handle); buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); if (buf->trbe_limit == buf->trbe_base) { trbe_drain_and_disable_local(); return; } trbe_enable_hw(buf); } static void trbe_handle_overflow(struct perf_output_handle *handle) { struct perf_event *event = handle->event; struct trbe_buf *buf = etm_perf_sink_config(handle); unsigned long offset, size; struct etm_event_data *event_data; offset = get_trbe_limit_pointer() - get_trbe_base_pointer(); size = offset - PERF_IDX2OFF(handle->head, buf); if (buf->snapshot) handle->head += size; /* * Mark the buffer as truncated, as we have stopped the trace * collection upon the WRAP event, without stopping the source. */ perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW | PERF_AUX_FLAG_TRUNCATED); perf_aux_output_end(handle, size); event_data = perf_aux_output_begin(handle, event); if (!event_data) { /* * We are unable to restart the trace collection, * thus leave the TRBE disabled. The etm-perf driver * is able to detect this with a disconnected handle * (handle->event = NULL). */ trbe_drain_and_disable_local(); *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; return; } buf->trbe_limit = compute_trbe_buffer_limit(handle); buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); if (buf->trbe_limit == buf->trbe_base) { trbe_stop_and_truncate_event(handle); return; } *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle; trbe_enable_hw(buf); } static bool is_perf_trbe(struct perf_output_handle *handle) { struct trbe_buf *buf = etm_perf_sink_config(handle); struct trbe_cpudata *cpudata = buf->cpudata; struct trbe_drvdata *drvdata = cpudata->drvdata; int cpu = smp_processor_id(); WARN_ON(buf->trbe_base != get_trbe_base_pointer()); WARN_ON(buf->trbe_limit != get_trbe_limit_pointer()); if (cpudata->mode != CS_MODE_PERF) return false; if (cpudata->cpu != cpu) return false; if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus)) return false; return true; } static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) { struct perf_output_handle **handle_ptr = dev; struct perf_output_handle *handle = *handle_ptr; enum trbe_fault_action act; u64 status; /* * Ensure the trace is visible to the CPUs and * any external aborts have been resolved. */ trbe_drain_and_disable_local(); status = read_sysreg_s(SYS_TRBSR_EL1); /* * If the pending IRQ was handled by update_buffer callback * we have nothing to do here. */ if (!is_trbe_irq(status)) return IRQ_NONE; clr_trbe_irq(); isb(); if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle)) return IRQ_NONE; if (!is_perf_trbe(handle)) return IRQ_NONE; /* * Ensure perf callbacks have completed, which may disable * the trace buffer in response to a TRUNCATION flag. */ irq_work_run(); act = trbe_get_fault_act(status); switch (act) { case TRBE_FAULT_ACT_WRAP: trbe_handle_overflow(handle); break; case TRBE_FAULT_ACT_SPURIOUS: trbe_handle_spurious(handle); break; case TRBE_FAULT_ACT_FATAL: trbe_stop_and_truncate_event(handle); break; } return IRQ_HANDLED; } static const struct coresight_ops_sink arm_trbe_sink_ops = { .enable = arm_trbe_enable, .disable = arm_trbe_disable, .alloc_buffer = arm_trbe_alloc_buffer, .free_buffer = arm_trbe_free_buffer, .update_buffer = arm_trbe_update_buffer, }; static const struct coresight_ops arm_trbe_cs_ops = { .sink_ops = &arm_trbe_sink_ops, }; static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf) { struct trbe_cpudata *cpudata = dev_get_drvdata(dev); return sprintf(buf, "%llx\n", cpudata->trbe_align); } static DEVICE_ATTR_RO(align); static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf) { struct trbe_cpudata *cpudata = dev_get_drvdata(dev); return sprintf(buf, "%d\n", cpudata->trbe_flag); } static DEVICE_ATTR_RO(flag); static struct attribute *arm_trbe_attrs[] = { &dev_attr_align.attr, &dev_attr_flag.attr, NULL, }; static const struct attribute_group arm_trbe_group = { .attrs = arm_trbe_attrs, }; static const struct attribute_group *arm_trbe_groups[] = { &arm_trbe_group, NULL, }; static void arm_trbe_enable_cpu(void *info) { struct trbe_drvdata *drvdata = info; trbe_reset_local(); enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE); } static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu) { struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu); struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu); struct coresight_desc desc = { 0 }; struct device *dev; if (WARN_ON(trbe_csdev)) return; dev = &cpudata->drvdata->pdev->dev; desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu); if (!desc.name) goto cpu_clear; desc.type = CORESIGHT_DEV_TYPE_SINK; desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM; desc.ops = &arm_trbe_cs_ops; desc.pdata = dev_get_platdata(dev); desc.groups = arm_trbe_groups; desc.dev = dev; trbe_csdev = coresight_register(&desc); if (IS_ERR(trbe_csdev)) goto cpu_clear; dev_set_drvdata(&trbe_csdev->dev, cpudata); coresight_set_percpu_sink(cpu, trbe_csdev); return; cpu_clear: cpumask_clear_cpu(cpu, &drvdata->supported_cpus); } static void arm_trbe_probe_cpu(void *info) { struct trbe_drvdata *drvdata = info; int cpu = smp_processor_id(); struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu); u64 trbidr; if (WARN_ON(!cpudata)) goto cpu_clear; if (!is_trbe_available()) { pr_err("TRBE is not implemented on cpu %d\n", cpu); goto cpu_clear; } trbidr = read_sysreg_s(SYS_TRBIDR_EL1); if (!is_trbe_programmable(trbidr)) { pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu); goto cpu_clear; } cpudata->trbe_align = 1ULL << get_trbe_address_align(trbidr); if (cpudata->trbe_align > SZ_2K) { pr_err("Unsupported alignment on cpu %d\n", cpu); goto cpu_clear; } cpudata->trbe_flag = get_trbe_flag_update(trbidr); cpudata->cpu = cpu; cpudata->drvdata = drvdata; return; cpu_clear: cpumask_clear_cpu(cpu, &drvdata->supported_cpus); } static void arm_trbe_remove_coresight_cpu(void *info) { int cpu = smp_processor_id(); struct trbe_drvdata *drvdata = info; struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu); struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu); disable_percpu_irq(drvdata->irq); trbe_reset_local(); if (trbe_csdev) { coresight_unregister(trbe_csdev); cpudata->drvdata = NULL; coresight_set_percpu_sink(cpu, NULL); } } static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata) { int cpu; drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata)); if (!drvdata->cpudata) return -ENOMEM; for_each_cpu(cpu, &drvdata->supported_cpus) { smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) arm_trbe_register_coresight_cpu(drvdata, cpu); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1); } return 0; } static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata) { int cpu; for_each_cpu(cpu, &drvdata->supported_cpus) smp_call_function_single(cpu, arm_trbe_remove_coresight_cpu, drvdata, 1); free_percpu(drvdata->cpudata); return 0; } static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node) { struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) { /* * If this CPU was not probed for TRBE, * initialize it now. */ if (!coresight_get_percpu_sink(cpu)) { arm_trbe_probe_cpu(drvdata); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) arm_trbe_register_coresight_cpu(drvdata, cpu); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) arm_trbe_enable_cpu(drvdata); } else { arm_trbe_enable_cpu(drvdata); } } return 0; } static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node) { struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) { disable_percpu_irq(drvdata->irq); trbe_reset_local(); } return 0; } static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata) { enum cpuhp_state trbe_online; int ret; trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, arm_trbe_cpu_startup, arm_trbe_cpu_teardown); if (trbe_online < 0) return trbe_online; ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node); if (ret) { cpuhp_remove_multi_state(trbe_online); return ret; } drvdata->trbe_online = trbe_online; return 0; } static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata) { cpuhp_remove_multi_state(drvdata->trbe_online); } static int arm_trbe_probe_irq(struct platform_device *pdev, struct trbe_drvdata *drvdata) { int ret; drvdata->irq = platform_get_irq(pdev, 0); if (drvdata->irq < 0) { pr_err("IRQ not found for the platform device\n"); return drvdata->irq; } if (!irq_is_percpu(drvdata->irq)) { pr_err("IRQ is not a PPI\n"); return -EINVAL; } if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus)) return -EINVAL; drvdata->handle = alloc_percpu(struct perf_output_handle *); if (!drvdata->handle) return -ENOMEM; ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle); if (ret) { free_percpu(drvdata->handle); return ret; } return 0; } static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata) { free_percpu_irq(drvdata->irq, drvdata->handle); free_percpu(drvdata->handle); } static int arm_trbe_device_probe(struct platform_device *pdev) { struct coresight_platform_data *pdata; struct trbe_drvdata *drvdata; struct device *dev = &pdev->dev; int ret; drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) return -ENOMEM; pdata = coresight_get_platform_data(dev); if (IS_ERR(pdata)) return PTR_ERR(pdata); dev_set_drvdata(dev, drvdata); dev->platform_data = pdata; drvdata->pdev = pdev; ret = arm_trbe_probe_irq(pdev, drvdata); if (ret) return ret; ret = arm_trbe_probe_coresight(drvdata); if (ret) goto probe_failed; ret = arm_trbe_probe_cpuhp(drvdata); if (ret) goto cpuhp_failed; return 0; cpuhp_failed: arm_trbe_remove_coresight(drvdata); probe_failed: arm_trbe_remove_irq(drvdata); return ret; } static int arm_trbe_device_remove(struct platform_device *pdev) { struct trbe_drvdata *drvdata = platform_get_drvdata(pdev); arm_trbe_remove_cpuhp(drvdata); arm_trbe_remove_coresight(drvdata); arm_trbe_remove_irq(drvdata); return 0; } static const struct of_device_id arm_trbe_of_match[] = { { .compatible = "arm,trace-buffer-extension"}, {}, }; MODULE_DEVICE_TABLE(of, arm_trbe_of_match); static struct platform_driver arm_trbe_driver = { .driver = { .name = DRVNAME, .of_match_table = of_match_ptr(arm_trbe_of_match), .suppress_bind_attrs = true, }, .probe = arm_trbe_device_probe, .remove = arm_trbe_device_remove, }; static int __init arm_trbe_init(void) { int ret; if (arm64_kernel_unmapped_at_el0()) { pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n"); return -EOPNOTSUPP; } ret = platform_driver_register(&arm_trbe_driver); if (!ret) return 0; pr_err("Error registering %s platform driver\n", DRVNAME); return ret; } static void __exit arm_trbe_exit(void) { platform_driver_unregister(&arm_trbe_driver); } module_init(arm_trbe_init); module_exit(arm_trbe_exit); MODULE_AUTHOR("Anshuman Khandual "); MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver"); MODULE_LICENSE("GPL v2");