summaryrefslogtreecommitdiff
path: root/drivers/perf
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/perf')
-rw-r--r--drivers/perf/Kconfig11
-rw-r--r--drivers/perf/Makefile1
-rw-r--r--drivers/perf/arm-cmn.c20
-rw-r--r--drivers/perf/arm-ni.c153
-rw-r--r--drivers/perf/arm_brbe.c805
-rw-r--r--drivers/perf/arm_brbe.h47
-rw-r--r--drivers/perf/arm_pmu.c16
-rw-r--r--drivers/perf/arm_pmuv3.c107
-rw-r--r--drivers/perf/arm_spe_pmu.c18
-rw-r--r--drivers/perf/cxl_pmu.c12
-rw-r--r--drivers/perf/fsl_imx9_ddr_perf.c8
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c354
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_hha_pmu.c6
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_pa_pmu.c2
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_pmu.c11
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_pmu.h2
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c220
17 files changed, 1442 insertions, 351 deletions
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 278c929dc87a..a9188dec36fe 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -223,6 +223,17 @@ config ARM_SPE_PMU
Extension, which provides periodic sampling of operations in
the CPU pipeline and reports this via the perf AUX interface.
+config ARM64_BRBE
+ bool "Enable support for branch stack sampling using FEAT_BRBE"
+ depends on ARM_PMUV3 && ARM64
+ default y
+ help
+ Enable perf support for Branch Record Buffer Extension (BRBE) which
+ records all branches taken in an execution path. This supports some
+ branch types and privilege based filtering. It captures additional
+ relevant information such as cycle count, misprediction and branch
+ type, branch privilege level etc.
+
config ARM_DMC620_PMU
tristate "Enable PMU support for the ARM DMC-620 memory controller"
depends on (ARM64 && ACPI) || COMPILE_TEST
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index de71d2574857..192fc8b16204 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -23,6 +23,7 @@ obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o
+obj-$(CONFIG_ARM64_BRBE) += arm_brbe.o
obj-$(CONFIG_ARM_DMC620_PMU) += arm_dmc620_pmu.o
obj-$(CONFIG_MARVELL_CN10K_TAD_PMU) += marvell_cn10k_tad_pmu.o
obj-$(CONFIG_MARVELL_CN10K_DDR_PMU) += marvell_cn10k_ddr_pmu.o
diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c
index 031d45d0fe3d..11fb2234b10f 100644
--- a/drivers/perf/arm-cmn.c
+++ b/drivers/perf/arm-cmn.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2016-2020 Arm Limited
-// CMN-600 Coherent Mesh Network PMU driver
+// ARM CMN/CI interconnect PMU driver
#include <linux/acpi.h>
#include <linux/bitfield.h>
@@ -2245,12 +2245,11 @@ static enum cmn_node_type arm_cmn_subtype(enum cmn_node_type type)
static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)
{
- void __iomem *cfg_region;
+ void __iomem *cfg_region, __iomem *xp_region;
struct arm_cmn_node cfg, *dn;
struct arm_cmn_dtm *dtm;
enum cmn_part part;
u16 child_count, child_poff;
- u32 xp_offset[CMN_MAX_XPS];
u64 reg;
int i, j;
size_t sz;
@@ -2302,11 +2301,12 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)
cmn->num_dns = cmn->num_xps;
/* Pass 1: visit the XPs, enumerate their children */
+ cfg_region += child_poff;
for (i = 0; i < cmn->num_xps; i++) {
- reg = readq_relaxed(cfg_region + child_poff + i * 8);
- xp_offset[i] = reg & CMN_CHILD_NODE_ADDR;
+ reg = readq_relaxed(cfg_region + i * 8);
+ xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR);
- reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO);
+ reg = readq_relaxed(xp_region + CMN_CHILD_INFO);
cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg);
}
@@ -2332,11 +2332,12 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)
cmn->dns = dn;
cmn->dtms = dtm;
for (i = 0; i < cmn->num_xps; i++) {
- void __iomem *xp_region = cmn->base + xp_offset[i];
struct arm_cmn_node *xp = dn++;
unsigned int xp_ports = 0;
- arm_cmn_init_node_info(cmn, xp_offset[i], xp);
+ reg = readq_relaxed(cfg_region + i * 8);
+ xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR);
+ arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, xp);
/*
* Thanks to the order in which XP logical IDs seem to be
* assigned, we can handily infer the mesh X dimension by
@@ -2655,6 +2656,7 @@ static struct platform_driver arm_cmn_driver = {
.name = "arm-cmn",
.of_match_table = of_match_ptr(arm_cmn_of_match),
.acpi_match_table = ACPI_PTR(arm_cmn_acpi_match),
+ .suppress_bind_attrs = true,
},
.probe = arm_cmn_probe,
.remove = arm_cmn_remove,
@@ -2693,5 +2695,5 @@ module_init(arm_cmn_init);
module_exit(arm_cmn_exit);
MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
-MODULE_DESCRIPTION("Arm CMN-600 PMU driver");
+MODULE_DESCRIPTION("Arm CMN/CI interconnect PMU driver");
MODULE_LICENSE("GPL v2");
diff --git a/drivers/perf/arm-ni.c b/drivers/perf/arm-ni.c
index de7b6cce4d68..1615a0564031 100644
--- a/drivers/perf/arm-ni.c
+++ b/drivers/perf/arm-ni.c
@@ -102,10 +102,9 @@ struct arm_ni_unit {
struct arm_ni_cd {
void __iomem *pmu_base;
u16 id;
+ s8 irq_friend;
int num_units;
int irq;
- int cpu;
- struct hlist_node cpuhp_node;
struct pmu pmu;
struct arm_ni_unit *units;
struct perf_event *evcnt[NI_NUM_COUNTERS];
@@ -117,13 +116,18 @@ struct arm_ni {
void __iomem *base;
enum ni_part part;
int id;
+ int cpu;
int num_cds;
+ struct hlist_node cpuhp_node;
struct arm_ni_cd cds[] __counted_by(num_cds);
};
#define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id])
#define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu)
+#define ni_for_each_cd(n, c) \
+ for (struct arm_ni_cd *c = n->cds; c < n->cds + n->num_cds; c++) if (c->pmu_base)
+
#define cd_for_each_unit(cd, u) \
for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++)
@@ -218,9 +222,9 @@ static const struct attribute_group arm_ni_format_attrs_group = {
static ssize_t arm_ni_cpumask_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
- struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));
+ struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
- return cpumap_print_to_pagebuf(true, buf, cpumask_of(cd->cpu));
+ return cpumap_print_to_pagebuf(true, buf, cpumask_of(ni->cpu));
}
static struct device_attribute arm_ni_cpumask_attr =
@@ -314,7 +318,7 @@ static int arm_ni_event_init(struct perf_event *event)
if (is_sampling_event(event))
return -EINVAL;
- event->cpu = cd->cpu;
+ event->cpu = cd_to_ni(cd)->cpu;
if (NI_EVENT_TYPE(event) == NI_PMU)
return arm_ni_validate_group(event);
@@ -445,33 +449,37 @@ static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id)
{
struct arm_ni_cd *cd = dev_id;
irqreturn_t ret = IRQ_NONE;
- u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);
- if (reg & (1U << NI_CCNT_IDX)) {
- ret = IRQ_HANDLED;
- if (!(WARN_ON(!cd->ccnt))) {
- arm_ni_event_read(cd->ccnt);
- arm_ni_init_ccnt(cd);
+ for (;;) {
+ u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);
+
+ if (reg & (1U << NI_CCNT_IDX)) {
+ ret = IRQ_HANDLED;
+ if (!(WARN_ON(!cd->ccnt))) {
+ arm_ni_event_read(cd->ccnt);
+ arm_ni_init_ccnt(cd);
+ }
}
- }
- for (int i = 0; i < NI_NUM_COUNTERS; i++) {
- if (!(reg & (1U << i)))
- continue;
- ret = IRQ_HANDLED;
- if (!(WARN_ON(!cd->evcnt[i]))) {
- arm_ni_event_read(cd->evcnt[i]);
- arm_ni_init_evcnt(cd, i);
+ for (int i = 0; i < NI_NUM_COUNTERS; i++) {
+ if (!(reg & (1U << i)))
+ continue;
+ ret = IRQ_HANDLED;
+ if (!(WARN_ON(!cd->evcnt[i]))) {
+ arm_ni_event_read(cd->evcnt[i]);
+ arm_ni_init_evcnt(cd, i);
+ }
}
+ writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);
+ if (!cd->irq_friend)
+ return ret;
+ cd += cd->irq_friend;
}
- writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);
- return ret;
}
static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start)
{
struct arm_ni_cd *cd = ni->cds + node->id;
const char *name;
- int err;
cd->id = node->id;
cd->num_units = node->num_components;
@@ -531,19 +539,11 @@ static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_s
cd->pmu_base + NI_PMCR);
writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR);
writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR);
- writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);
cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id);
if (cd->irq < 0)
return cd->irq;
- err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,
- IRQF_NOBALANCING | IRQF_NO_THREAD,
- dev_name(ni->dev), cd);
- if (err)
- return err;
-
- cd->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));
cd->pmu = (struct pmu) {
.module = THIS_MODULE,
.parent = ni->dev,
@@ -564,32 +564,19 @@ static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_s
if (!name)
return -ENOMEM;
- err = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);
- if (err)
- return err;
-
- err = perf_pmu_register(&cd->pmu, name, -1);
- if (err)
- cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);
-
- return err;
+ return perf_pmu_register(&cd->pmu, name, -1);
}
static void arm_ni_remove(struct platform_device *pdev)
{
struct arm_ni *ni = platform_get_drvdata(pdev);
- for (int i = 0; i < ni->num_cds; i++) {
- struct arm_ni_cd *cd = ni->cds + i;
-
- if (!cd->pmu_base)
- continue;
-
+ ni_for_each_cd(ni, cd) {
writel_relaxed(0, cd->pmu_base + NI_PMCR);
writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR);
perf_pmu_unregister(&cd->pmu);
- cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);
}
+ cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
}
static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node)
@@ -602,6 +589,34 @@ static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node)
node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO);
}
+static int arm_ni_init_irqs(struct arm_ni *ni)
+{
+ int err;
+
+ ni_for_each_cd(ni, cd) {
+ for (struct arm_ni_cd *prev = cd; prev-- > ni->cds; ) {
+ if (prev->irq == cd->irq) {
+ prev->irq_friend = cd - prev;
+ goto set_inten;
+ }
+ }
+ err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,
+ IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN,
+ dev_name(ni->dev), cd);
+ if (err)
+ return err;
+
+ irq_set_affinity(cd->irq, cpumask_of(ni->cpu));
+set_inten:
+ writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);
+ }
+
+ ni_for_each_cd(ni, cd)
+ if (!cd->irq_friend)
+ enable_irq(cd->irq);
+ return 0;
+}
+
static int arm_ni_probe(struct platform_device *pdev)
{
struct arm_ni_node cfg, vd, pd, cd;
@@ -609,7 +624,7 @@ static int arm_ni_probe(struct platform_device *pdev)
struct resource *res;
void __iomem *base;
static atomic_t id;
- int num_cds;
+ int ret, num_cds;
u32 reg, part;
/*
@@ -660,8 +675,13 @@ static int arm_ni_probe(struct platform_device *pdev)
ni->num_cds = num_cds;
ni->part = part;
ni->id = atomic_fetch_inc(&id);
+ ni->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));
platform_set_drvdata(pdev, ni);
+ ret = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
+ if (ret)
+ return ret;
+
for (int v = 0; v < cfg.num_components; v++) {
reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
arm_ni_probe_domain(base + reg, &vd);
@@ -669,8 +689,6 @@ static int arm_ni_probe(struct platform_device *pdev)
reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
arm_ni_probe_domain(base + reg, &pd);
for (int c = 0; c < pd.num_components; c++) {
- int ret;
-
reg = readl_relaxed(pd.base + NI_CHILD_PTR(c));
arm_ni_probe_domain(base + reg, &cd);
ret = arm_ni_init_cd(ni, &cd, res->start);
@@ -683,7 +701,11 @@ static int arm_ni_probe(struct platform_device *pdev)
}
}
- return 0;
+ ret = arm_ni_init_irqs(ni);
+ if (ret)
+ arm_ni_remove(pdev);
+
+ return ret;
}
#ifdef CONFIG_OF
@@ -707,47 +729,50 @@ static struct platform_driver arm_ni_driver = {
.name = "arm-ni",
.of_match_table = of_match_ptr(arm_ni_of_match),
.acpi_match_table = ACPI_PTR(arm_ni_acpi_match),
+ .suppress_bind_attrs = true,
},
.probe = arm_ni_probe,
.remove = arm_ni_remove,
};
-static void arm_ni_pmu_migrate(struct arm_ni_cd *cd, unsigned int cpu)
+static void arm_ni_pmu_migrate(struct arm_ni *ni, unsigned int cpu)
{
- perf_pmu_migrate_context(&cd->pmu, cd->cpu, cpu);
- irq_set_affinity(cd->irq, cpumask_of(cpu));
- cd->cpu = cpu;
+ ni_for_each_cd(ni, cd) {
+ perf_pmu_migrate_context(&cd->pmu, ni->cpu, cpu);
+ irq_set_affinity(cd->irq, cpumask_of(cpu));
+ }
+ ni->cpu = cpu;
}
static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
{
- struct arm_ni_cd *cd;
+ struct arm_ni *ni;
int node;
- cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);
- node = dev_to_node(cd_to_ni(cd)->dev);
- if (cpu_to_node(cd->cpu) != node && cpu_to_node(cpu) == node)
- arm_ni_pmu_migrate(cd, cpu);
+ ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
+ node = dev_to_node(ni->dev);
+ if (cpu_to_node(ni->cpu) != node && cpu_to_node(cpu) == node)
+ arm_ni_pmu_migrate(ni, cpu);
return 0;
}
static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
{
- struct arm_ni_cd *cd;
+ struct arm_ni *ni;
unsigned int target;
int node;
- cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);
- if (cpu != cd->cpu)
+ ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
+ if (cpu != ni->cpu)
return 0;
- node = dev_to_node(cd_to_ni(cd)->dev);
+ node = dev_to_node(ni->dev);
target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
if (target >= nr_cpu_ids)
target = cpumask_any_but(cpu_online_mask, cpu);
if (target < nr_cpu_ids)
- arm_ni_pmu_migrate(cd, target);
+ arm_ni_pmu_migrate(ni, target);
return 0;
}
diff --git a/drivers/perf/arm_brbe.c b/drivers/perf/arm_brbe.c
new file mode 100644
index 000000000000..ba554e0c846c
--- /dev/null
+++ b/drivers/perf/arm_brbe.c
@@ -0,0 +1,805 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Branch Record Buffer Extension Driver.
+ *
+ * Copyright (C) 2022-2025 ARM Limited
+ *
+ * Author: Anshuman Khandual <anshuman.khandual@arm.com>
+ */
+#include <linux/types.h>
+#include <linux/bitmap.h>
+#include <linux/perf/arm_pmu.h>
+#include "arm_brbe.h"
+
+#define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \
+ BRBFCR_EL1_INDIRECT | \
+ BRBFCR_EL1_RTN | \
+ BRBFCR_EL1_INDCALL | \
+ BRBFCR_EL1_DIRCALL | \
+ BRBFCR_EL1_CONDDIR)
+
+/*
+ * BRBTS_EL1 is currently not used for branch stack implementation
+ * purpose but BRBCR_ELx.TS needs to have a valid value from all
+ * available options. BRBCR_ELx_TS_VIRTUAL is selected for this.
+ */
+#define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL)
+
+/*
+ * BRBE Buffer Organization
+ *
+ * BRBE buffer is arranged as multiple banks of 32 branch record
+ * entries each. An individual branch record in a given bank could
+ * be accessed, after selecting the bank in BRBFCR_EL1.BANK and
+ * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with
+ * indices [0..31].
+ *
+ * Bank 0
+ *
+ * --------------------------------- ------
+ * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 |
+ * --------------------------------- ------
+ * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 |
+ * --------------------------------- ------
+ * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
+ * --------------------------------- ------
+ * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 |
+ * --------------------------------- ------
+ *
+ * Bank 1
+ *
+ * --------------------------------- ------
+ * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 |
+ * --------------------------------- ------
+ * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 |
+ * --------------------------------- ------
+ * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
+ * --------------------------------- ------
+ * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 |
+ * --------------------------------- ------
+ */
+#define BRBE_BANK_MAX_ENTRIES 32
+
+struct brbe_regset {
+ u64 brbsrc;
+ u64 brbtgt;
+ u64 brbinf;
+};
+
+#define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX)
+
+struct brbe_hw_attr {
+ int brbe_version;
+ int brbe_cc;
+ int brbe_nr;
+ int brbe_format;
+};
+
+#define BRBE_REGN_CASE(n, case_macro) \
+ case n: case_macro(n); break
+
+#define BRBE_REGN_SWITCH(x, case_macro) \
+ do { \
+ switch (x) { \
+ BRBE_REGN_CASE(0, case_macro); \
+ BRBE_REGN_CASE(1, case_macro); \
+ BRBE_REGN_CASE(2, case_macro); \
+ BRBE_REGN_CASE(3, case_macro); \
+ BRBE_REGN_CASE(4, case_macro); \
+ BRBE_REGN_CASE(5, case_macro); \
+ BRBE_REGN_CASE(6, case_macro); \
+ BRBE_REGN_CASE(7, case_macro); \
+ BRBE_REGN_CASE(8, case_macro); \
+ BRBE_REGN_CASE(9, case_macro); \
+ BRBE_REGN_CASE(10, case_macro); \
+ BRBE_REGN_CASE(11, case_macro); \
+ BRBE_REGN_CASE(12, case_macro); \
+ BRBE_REGN_CASE(13, case_macro); \
+ BRBE_REGN_CASE(14, case_macro); \
+ BRBE_REGN_CASE(15, case_macro); \
+ BRBE_REGN_CASE(16, case_macro); \
+ BRBE_REGN_CASE(17, case_macro); \
+ BRBE_REGN_CASE(18, case_macro); \
+ BRBE_REGN_CASE(19, case_macro); \
+ BRBE_REGN_CASE(20, case_macro); \
+ BRBE_REGN_CASE(21, case_macro); \
+ BRBE_REGN_CASE(22, case_macro); \
+ BRBE_REGN_CASE(23, case_macro); \
+ BRBE_REGN_CASE(24, case_macro); \
+ BRBE_REGN_CASE(25, case_macro); \
+ BRBE_REGN_CASE(26, case_macro); \
+ BRBE_REGN_CASE(27, case_macro); \
+ BRBE_REGN_CASE(28, case_macro); \
+ BRBE_REGN_CASE(29, case_macro); \
+ BRBE_REGN_CASE(30, case_macro); \
+ BRBE_REGN_CASE(31, case_macro); \
+ default: WARN(1, "Invalid BRB* index %d\n", x); \
+ } \
+ } while (0)
+
+#define RETURN_READ_BRBSRCN(n) \
+ return read_sysreg_s(SYS_BRBSRC_EL1(n))
+static inline u64 get_brbsrc_reg(int idx)
+{
+ BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN);
+ return 0;
+}
+
+#define RETURN_READ_BRBTGTN(n) \
+ return read_sysreg_s(SYS_BRBTGT_EL1(n))
+static u64 get_brbtgt_reg(int idx)
+{
+ BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN);
+ return 0;
+}
+
+#define RETURN_READ_BRBINFN(n) \
+ return read_sysreg_s(SYS_BRBINF_EL1(n))
+static u64 get_brbinf_reg(int idx)
+{
+ BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN);
+ return 0;
+}
+
+static u64 brbe_record_valid(u64 brbinf)
+{
+ return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf);
+}
+
+static bool brbe_invalid(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE;
+}
+
+static bool brbe_record_is_complete(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL;
+}
+
+static bool brbe_record_is_source_only(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE;
+}
+
+static bool brbe_record_is_target_only(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET;
+}
+
+static int brbinf_get_in_tx(u64 brbinf)
+{
+ return FIELD_GET(BRBINFx_EL1_T_MASK, brbinf);
+}
+
+static int brbinf_get_mispredict(u64 brbinf)
+{
+ return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf);
+}
+
+static int brbinf_get_lastfailed(u64 brbinf)
+{
+ return FIELD_GET(BRBINFx_EL1_LASTFAILED_MASK, brbinf);
+}
+
+static u16 brbinf_get_cycles(u64 brbinf)
+{
+ u32 exp, mant, cycles;
+ /*
+ * Captured cycle count is unknown and hence
+ * should not be passed on to userspace.
+ */
+ if (brbinf & BRBINFx_EL1_CCU)
+ return 0;
+
+ exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf);
+ mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf);
+
+ if (!exp)
+ return mant;
+
+ cycles = (mant | 0x100) << (exp - 1);
+
+ return min(cycles, U16_MAX);
+}
+
+static int brbinf_get_type(u64 brbinf)
+{
+ return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf);
+}
+
+static int brbinf_get_el(u64 brbinf)
+{
+ return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf);
+}
+
+void brbe_invalidate(void)
+{
+ /* Ensure all branches before this point are recorded */
+ isb();
+ asm volatile(BRB_IALL_INSN);
+ /* Ensure all branch records are invalidated after this point */
+ isb();
+}
+
+static bool valid_brbe_nr(int brbe_nr)
+{
+ return brbe_nr == BRBIDR0_EL1_NUMREC_8 ||
+ brbe_nr == BRBIDR0_EL1_NUMREC_16 ||
+ brbe_nr == BRBIDR0_EL1_NUMREC_32 ||
+ brbe_nr == BRBIDR0_EL1_NUMREC_64;
+}
+
+static bool valid_brbe_cc(int brbe_cc)
+{
+ return brbe_cc == BRBIDR0_EL1_CC_20_BIT;
+}
+
+static bool valid_brbe_format(int brbe_format)
+{
+ return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0;
+}
+
+static bool valid_brbidr(u64 brbidr)
+{
+ int brbe_format, brbe_cc, brbe_nr;
+
+ brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr);
+ brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr);
+ brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr);
+
+ return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr);
+}
+
+static bool valid_brbe_version(int brbe_version)
+{
+ return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP ||
+ brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1;
+}
+
+static void select_brbe_bank(int bank)
+{
+ u64 brbfcr;
+
+ brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
+ brbfcr &= ~BRBFCR_EL1_BANK_MASK;
+ brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank);
+ write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
+ /*
+ * Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync
+ * between setting BANK and accessing branch records.
+ */
+ isb();
+}
+
+static bool __read_brbe_regset(struct brbe_regset *entry, int idx)
+{
+ entry->brbinf = get_brbinf_reg(idx);
+
+ if (brbe_invalid(entry->brbinf))
+ return false;
+
+ entry->brbsrc = get_brbsrc_reg(idx);
+ entry->brbtgt = get_brbtgt_reg(idx);
+ return true;
+}
+
+/*
+ * Generic perf branch filters supported on BRBE
+ *
+ * New branch filters need to be evaluated whether they could be supported on
+ * BRBE. This ensures that such branch filters would not just be accepted, to
+ * fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively
+ * supported only on platforms where kernel is in hyp mode.
+ */
+#define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \
+ PERF_SAMPLE_BRANCH_IN_TX | \
+ PERF_SAMPLE_BRANCH_NO_TX | \
+ PERF_SAMPLE_BRANCH_CALL_STACK | \
+ PERF_SAMPLE_BRANCH_COUNTERS)
+
+#define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \
+ PERF_SAMPLE_BRANCH_ANY_CALL | \
+ PERF_SAMPLE_BRANCH_ANY_RETURN | \
+ PERF_SAMPLE_BRANCH_IND_CALL | \
+ PERF_SAMPLE_BRANCH_COND | \
+ PERF_SAMPLE_BRANCH_IND_JUMP | \
+ PERF_SAMPLE_BRANCH_CALL)
+
+
+#define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \
+ PERF_SAMPLE_BRANCH_KERNEL | \
+ PERF_SAMPLE_BRANCH_HV | \
+ BRBE_ALLOWED_BRANCH_TYPES | \
+ PERF_SAMPLE_BRANCH_NO_FLAGS | \
+ PERF_SAMPLE_BRANCH_NO_CYCLES | \
+ PERF_SAMPLE_BRANCH_TYPE_SAVE | \
+ PERF_SAMPLE_BRANCH_HW_INDEX | \
+ PERF_SAMPLE_BRANCH_PRIV_SAVE)
+
+#define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \
+ BRBE_EXCLUDE_BRANCH_FILTERS)
+
+/*
+ * BRBE supports the following functional branch type filters while
+ * generating branch records. These branch filters can be enabled,
+ * either individually or as a group i.e ORing multiple filters
+ * with each other.
+ *
+ * BRBFCR_EL1_CONDDIR - Conditional direct branch
+ * BRBFCR_EL1_DIRCALL - Direct call
+ * BRBFCR_EL1_INDCALL - Indirect call
+ * BRBFCR_EL1_INDIRECT - Indirect branch
+ * BRBFCR_EL1_DIRECT - Direct branch
+ * BRBFCR_EL1_RTN - Subroutine return
+ */
+static u64 branch_type_to_brbfcr(int branch_type)
+{
+ u64 brbfcr = 0;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
+ brbfcr |= BRBFCR_EL1_BRANCH_FILTERS;
+ return brbfcr;
+ }
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) {
+ brbfcr |= BRBFCR_EL1_INDCALL;
+ brbfcr |= BRBFCR_EL1_DIRCALL;
+ }
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
+ brbfcr |= BRBFCR_EL1_RTN;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL)
+ brbfcr |= BRBFCR_EL1_INDCALL;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_COND)
+ brbfcr |= BRBFCR_EL1_CONDDIR;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP)
+ brbfcr |= BRBFCR_EL1_INDIRECT;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_CALL)
+ brbfcr |= BRBFCR_EL1_DIRCALL;
+
+ return brbfcr;
+}
+
+/*
+ * BRBE supports the following privilege mode filters while generating
+ * branch records.
+ *
+ * BRBCR_ELx_E0BRE - EL0 branch records
+ * BRBCR_ELx_ExBRE - EL1/EL2 branch records
+ *
+ * BRBE also supports the following additional functional branch type
+ * filters while generating branch records.
+ *
+ * BRBCR_ELx_EXCEPTION - Exception
+ * BRBCR_ELx_ERTN - Exception return
+ */
+static u64 branch_type_to_brbcr(int branch_type)
+{
+ u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_USER)
+ brbcr |= BRBCR_ELx_E0BRE;
+
+ /*
+ * When running in the hyp mode, writing into BRBCR_EL1
+ * actually writes into BRBCR_EL2 instead. Field E2BRE
+ * is also at the same position as E1BRE.
+ */
+ if (branch_type & PERF_SAMPLE_BRANCH_KERNEL)
+ brbcr |= BRBCR_ELx_ExBRE;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_HV) {
+ if (is_kernel_in_hyp_mode())
+ brbcr |= BRBCR_ELx_ExBRE;
+ }
+
+ if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES))
+ brbcr |= BRBCR_ELx_CC;
+
+ if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS))
+ brbcr |= BRBCR_ELx_MPRED;
+
+ /*
+ * The exception and exception return branches could be
+ * captured, irrespective of the perf event's privilege.
+ * If the perf event does not have enough privilege for
+ * a given exception level, then addresses which falls
+ * under that exception level will be reported as zero
+ * for the captured branch record, creating source only
+ * or target only records.
+ */
+ if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) {
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
+ brbcr |= BRBCR_ELx_EXCEPTION;
+ brbcr |= BRBCR_ELx_ERTN;
+ }
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL)
+ brbcr |= BRBCR_ELx_EXCEPTION;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
+ brbcr |= BRBCR_ELx_ERTN;
+ }
+ return brbcr;
+}
+
+bool brbe_branch_attr_valid(struct perf_event *event)
+{
+ u64 branch_type = event->attr.branch_sample_type;
+
+ /*
+ * Ensure both perf branch filter allowed and exclude
+ * masks are always in sync with the generic perf ABI.
+ */
+ BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1));
+
+ if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) {
+ pr_debug("requested branch filter not supported 0x%llx\n", branch_type);
+ return false;
+ }
+
+ /* Ensure at least 1 branch type is enabled */
+ if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) {
+ pr_debug("no branch type enabled 0x%llx\n", branch_type);
+ return false;
+ }
+
+ /*
+ * No branches are recorded in guests nor nVHE hypervisors, so
+ * excluding the host or both kernel and user is invalid.
+ *
+ * Ideally we'd just require exclude_guest and exclude_hv, but setting
+ * event filters with perf for kernel or user don't set exclude_guest.
+ * So effectively, exclude_guest and exclude_hv are ignored.
+ */
+ if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) {
+ pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type);
+ return false;
+ }
+
+ event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type);
+ event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type);
+
+ return true;
+}
+
+unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu)
+{
+ return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr);
+}
+
+void brbe_probe(struct arm_pmu *armpmu)
+{
+ u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
+ u32 brbe;
+
+ brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT);
+ if (!valid_brbe_version(brbe))
+ return;
+
+ brbidr = read_sysreg_s(SYS_BRBIDR0_EL1);
+ if (!valid_brbidr(brbidr))
+ return;
+
+ armpmu->reg_brbidr = brbidr;
+}
+
+/*
+ * BRBE is assumed to be disabled/paused on entry
+ */
+void brbe_enable(const struct arm_pmu *arm_pmu)
+{
+ struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events);
+ u64 brbfcr = 0, brbcr = 0;
+
+ /*
+ * Discard existing records to avoid a discontinuity, e.g. records
+ * missed during handling an overflow.
+ */
+ brbe_invalidate();
+
+ /*
+ * Merge the permitted branch filters of all events.
+ */
+ for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) {
+ struct perf_event *event = cpuc->events[i];
+
+ if (event && has_branch_stack(event)) {
+ brbfcr |= event->hw.branch_reg.config;
+ brbcr |= event->hw.extra_reg.config;
+ }
+ }
+
+ /*
+ * In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP
+ * controls freezing the branch records on counter overflow rather than
+ * BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to).
+ * The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1
+ * and EL0 recording disabled for guests.
+ *
+ * As BRBCR_EL1 CC and MPRED bits also need to match, use the same
+ * value for both registers just masking the exception levels.
+ */
+ if (is_kernel_in_hyp_mode())
+ write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12);
+ write_sysreg_s(brbcr, SYS_BRBCR_EL1);
+ /* Ensure BRBCR_ELx settings take effect before unpausing */
+ isb();
+
+ /* Finally write SYS_BRBFCR_EL to unpause BRBE */
+ write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
+ /* Synchronization in PMCR write ensures ordering WRT PMU enabling */
+}
+
+void brbe_disable(void)
+{
+ /*
+ * No need for synchronization here as synchronization in PMCR write
+ * ensures ordering and in the interrupt handler this is a NOP as
+ * we're already paused.
+ */
+ write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
+ write_sysreg_s(0, SYS_BRBCR_EL1);
+}
+
+static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = {
+ [BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 },
+ [BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 },
+ [BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 },
+ [BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 },
+ [BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 },
+ [BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 },
+ [BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 },
+ [BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 },
+ [BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 },
+ [BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 },
+ [BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 },
+ [BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN },
+ [BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST },
+ [BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA },
+};
+
+static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf)
+{
+ int brbe_type = brbinf_get_type(brbinf);
+
+ if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) {
+ const int *br_type = brbe_type_to_perf_type_map[brbe_type];
+
+ entry->type = br_type[0];
+ entry->new_type = br_type[1];
+ }
+}
+
+static int brbinf_get_perf_priv(u64 brbinf)
+{
+ int brbe_el = brbinf_get_el(brbinf);
+
+ switch (brbe_el) {
+ case BRBINFx_EL1_EL_EL0:
+ return PERF_BR_PRIV_USER;
+ case BRBINFx_EL1_EL_EL1:
+ return PERF_BR_PRIV_KERNEL;
+ case BRBINFx_EL1_EL_EL2:
+ if (is_kernel_in_hyp_mode())
+ return PERF_BR_PRIV_KERNEL;
+ return PERF_BR_PRIV_HV;
+ default:
+ pr_warn_once("%d - unknown branch privilege captured\n", brbe_el);
+ return PERF_BR_PRIV_UNKNOWN;
+ }
+}
+
+static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry,
+ const struct perf_event *event)
+{
+ struct brbe_regset bregs;
+ u64 brbinf;
+
+ if (!__read_brbe_regset(&bregs, index))
+ return false;
+
+ brbinf = bregs.brbinf;
+ perf_clear_branch_entry_bitfields(entry);
+ if (brbe_record_is_complete(brbinf)) {
+ entry->from = bregs.brbsrc;
+ entry->to = bregs.brbtgt;
+ } else if (brbe_record_is_source_only(brbinf)) {
+ entry->from = bregs.brbsrc;
+ entry->to = 0;
+ } else if (brbe_record_is_target_only(brbinf)) {
+ entry->from = 0;
+ entry->to = bregs.brbtgt;
+ }
+
+ brbe_set_perf_entry_type(entry, brbinf);
+
+ if (!branch_sample_no_cycles(event))
+ entry->cycles = brbinf_get_cycles(brbinf);
+
+ if (!branch_sample_no_flags(event)) {
+ /* Mispredict info is available for source only and complete branch records. */
+ if (!brbe_record_is_target_only(brbinf)) {
+ entry->mispred = brbinf_get_mispredict(brbinf);
+ entry->predicted = !entry->mispred;
+ }
+
+ /*
+ * Currently TME feature is neither implemented in any hardware
+ * nor it is being supported in the kernel. Just warn here once
+ * if TME related information shows up rather unexpectedly.
+ */
+ if (brbinf_get_lastfailed(brbinf) || brbinf_get_in_tx(brbinf))
+ pr_warn_once("Unknown transaction states\n");
+ }
+
+ /*
+ * Branch privilege level is available for target only and complete
+ * branch records.
+ */
+ if (!brbe_record_is_source_only(brbinf))
+ entry->priv = brbinf_get_perf_priv(brbinf);
+
+ return true;
+}
+
+#define PERF_BR_ARM64_ALL ( \
+ BIT(PERF_BR_COND) | \
+ BIT(PERF_BR_UNCOND) | \
+ BIT(PERF_BR_IND) | \
+ BIT(PERF_BR_CALL) | \
+ BIT(PERF_BR_IND_CALL) | \
+ BIT(PERF_BR_RET))
+
+#define PERF_BR_ARM64_ALL_KERNEL ( \
+ BIT(PERF_BR_SYSCALL) | \
+ BIT(PERF_BR_IRQ) | \
+ BIT(PERF_BR_SERROR) | \
+ BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \
+ BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \
+ BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST))
+
+static void prepare_event_branch_type_mask(u64 branch_sample,
+ unsigned long *event_type_mask)
+{
+ if (branch_sample & PERF_SAMPLE_BRANCH_ANY) {
+ if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
+ bitmap_from_u64(event_type_mask,
+ BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL |
+ PERF_BR_ARM64_ALL_KERNEL);
+ else
+ bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL);
+ return;
+ }
+
+ bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX);
+
+ if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) {
+ if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
+ bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL);
+
+ set_bit(PERF_BR_CALL, event_type_mask);
+ set_bit(PERF_BR_IND_CALL, event_type_mask);
+ }
+
+ if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP)
+ set_bit(PERF_BR_IND, event_type_mask);
+
+ if (branch_sample & PERF_SAMPLE_BRANCH_COND)
+ set_bit(PERF_BR_COND, event_type_mask);
+
+ if (branch_sample & PERF_SAMPLE_BRANCH_CALL)
+ set_bit(PERF_BR_CALL, event_type_mask);
+
+ if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL)
+ set_bit(PERF_BR_IND_CALL, event_type_mask);
+
+ if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) {
+ set_bit(PERF_BR_RET, event_type_mask);
+
+ if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
+ set_bit(PERF_BR_ERET, event_type_mask);
+ }
+}
+
+/*
+ * BRBE is configured with an OR of permissions from all events, so there may
+ * be events which have to be dropped or events where just the source or target
+ * address has to be zeroed.
+ */
+static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type)
+{
+ bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4);
+ bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4);
+ bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) ||
+ (is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV)));
+
+ /* We can only have a half record if permissions have not been expanded */
+ if (!entry->from || !entry->to)
+ return true;
+
+ /*
+ * If record is within a single exception level, just need to either
+ * drop or keep the entire record.
+ */
+ if (from_user == to_user)
+ return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) ||
+ ((entry->priv == PERF_BR_PRIV_USER) &&
+ (branch_sample_type & PERF_SAMPLE_BRANCH_USER));
+
+ /*
+ * Record is across exception levels, mask addresses for the exception
+ * level we're not capturing.
+ */
+ if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) {
+ if (from_user)
+ entry->from = 0;
+ if (to_user)
+ entry->to = 0;
+ }
+
+ if (exclude_kernel) {
+ if (!from_user)
+ entry->from = 0;
+ if (!to_user)
+ entry->to = 0;
+ }
+
+ return true;
+}
+
+static bool filter_branch_type(struct perf_branch_entry *entry,
+ const unsigned long *event_type_mask)
+{
+ if (entry->type == PERF_BR_EXTEND_ABI)
+ return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask);
+ else
+ return test_bit(entry->type, event_type_mask);
+}
+
+static bool filter_branch_record(struct perf_branch_entry *entry,
+ u64 branch_sample,
+ const unsigned long *event_type_mask)
+{
+ return filter_branch_type(entry, event_type_mask) &&
+ filter_branch_privilege(entry, branch_sample);
+}
+
+void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack,
+ const struct perf_event *event)
+{
+ struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
+ int nr_hw = brbe_num_branch_records(cpu_pmu);
+ int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES);
+ int nr_filtered = 0;
+ u64 branch_sample_type = event->attr.branch_sample_type;
+ DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX);
+
+ prepare_event_branch_type_mask(branch_sample_type, event_type_mask);
+
+ for (int bank = 0; bank < nr_banks; bank++) {
+ int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES);
+ int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES);
+
+ select_brbe_bank(bank);
+
+ for (int i = 0; i < nr_this_bank; i++) {
+ struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered];
+
+ if (!perf_entry_from_brbe_regset(i, pbe, event))
+ goto done;
+
+ if (!filter_branch_record(pbe, branch_sample_type, event_type_mask))
+ continue;
+
+ nr_filtered++;
+ }
+ }
+
+done:
+ branch_stack->nr = nr_filtered;
+}
diff --git a/drivers/perf/arm_brbe.h b/drivers/perf/arm_brbe.h
new file mode 100644
index 000000000000..b7c7d8796c86
--- /dev/null
+++ b/drivers/perf/arm_brbe.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Branch Record Buffer Extension Helpers.
+ *
+ * Copyright (C) 2022-2025 ARM Limited
+ *
+ * Author: Anshuman Khandual <anshuman.khandual@arm.com>
+ */
+
+struct arm_pmu;
+struct perf_branch_stack;
+struct perf_event;
+
+#ifdef CONFIG_ARM64_BRBE
+void brbe_probe(struct arm_pmu *arm_pmu);
+unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu);
+void brbe_invalidate(void);
+
+void brbe_enable(const struct arm_pmu *arm_pmu);
+void brbe_disable(void);
+
+bool brbe_branch_attr_valid(struct perf_event *event);
+void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack,
+ const struct perf_event *event);
+#else
+static inline void brbe_probe(struct arm_pmu *arm_pmu) { }
+static inline unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu)
+{
+ return 0;
+}
+
+static inline void brbe_invalidate(void) { }
+
+static inline void brbe_enable(const struct arm_pmu *arm_pmu) { };
+static inline void brbe_disable(void) { };
+
+static inline bool brbe_branch_attr_valid(struct perf_event *event)
+{
+ WARN_ON_ONCE(!has_branch_stack(event));
+ return false;
+}
+
+static void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack,
+ const struct perf_event *event)
+{
+}
+#endif
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 2f33e69a8caf..5c310e803dd7 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -99,7 +99,7 @@ static const struct pmu_irq_ops percpu_pmunmi_ops = {
.free_pmuirq = armpmu_free_percpu_pmunmi
};
-static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu);
+DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu);
static DEFINE_PER_CPU(int, cpu_irq);
static DEFINE_PER_CPU(const struct pmu_irq_ops *, cpu_irq_ops);
@@ -318,6 +318,12 @@ armpmu_del(struct perf_event *event, int flags)
int idx = hwc->idx;
armpmu_stop(event, PERF_EF_UPDATE);
+
+ if (has_branch_stack(event)) {
+ hw_events->branch_users--;
+ perf_sched_cb_dec(event->pmu);
+ }
+
hw_events->events[idx] = NULL;
armpmu->clear_event_idx(hw_events, event);
perf_event_update_userpage(event);
@@ -345,6 +351,11 @@ armpmu_add(struct perf_event *event, int flags)
/* The newly-allocated counter should be empty */
WARN_ON_ONCE(hw_events->events[idx]);
+ if (has_branch_stack(event)) {
+ hw_events->branch_users++;
+ perf_sched_cb_inc(event->pmu);
+ }
+
event->hw.idx = idx;
hw_events->events[idx] = event;
@@ -509,8 +520,7 @@ static int armpmu_event_init(struct perf_event *event)
!cpumask_test_cpu(event->cpu, &armpmu->supported_cpus))
return -ENOENT;
- /* does not support taken branch sampling */
- if (has_branch_stack(event))
+ if (has_branch_stack(event) && !armpmu->reg_brbidr)
return -EOPNOTSUPP;
return __hw_perf_event_init(event);
diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c
index 3db9f4ed17e8..f6d7bab5d555 100644
--- a/drivers/perf/arm_pmuv3.c
+++ b/drivers/perf/arm_pmuv3.c
@@ -25,6 +25,8 @@
#include <linux/smp.h>
#include <linux/nmi.h>
+#include "arm_brbe.h"
+
/* ARMv8 Cortex-A53 specific event types. */
#define ARMV8_A53_PERFCTR_PREF_LINEFILL 0xC2
@@ -438,7 +440,19 @@ static ssize_t threshold_max_show(struct device *dev,
static DEVICE_ATTR_RO(threshold_max);
+static ssize_t branches_show(struct device *dev,
+ struct device_attribute *attr, char *page)
+{
+ struct pmu *pmu = dev_get_drvdata(dev);
+ struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
+
+ return sysfs_emit(page, "%d\n", brbe_num_branch_records(cpu_pmu));
+}
+
+static DEVICE_ATTR_RO(branches);
+
static struct attribute *armv8_pmuv3_caps_attrs[] = {
+ &dev_attr_branches.attr,
&dev_attr_slots.attr,
&dev_attr_bus_slots.attr,
&dev_attr_bus_width.attr,
@@ -446,9 +460,22 @@ static struct attribute *armv8_pmuv3_caps_attrs[] = {
NULL,
};
+static umode_t caps_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+ struct device *dev = kobj_to_dev(kobj);
+ struct pmu *pmu = dev_get_drvdata(dev);
+ struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
+
+ if (i == 0)
+ return brbe_num_branch_records(cpu_pmu) ? attr->mode : 0;
+
+ return attr->mode;
+}
+
static const struct attribute_group armv8_pmuv3_caps_attr_group = {
.name = "caps",
.attrs = armv8_pmuv3_caps_attrs,
+ .is_visible = caps_is_visible,
};
/*
@@ -809,6 +836,7 @@ static void armv8pmu_disable_event(struct perf_event *event)
static void armv8pmu_start(struct arm_pmu *cpu_pmu)
{
struct perf_event_context *ctx;
+ struct pmu_hw_events *hw_events = this_cpu_ptr(cpu_pmu->hw_events);
int nr_user = 0;
ctx = perf_cpu_task_ctx();
@@ -822,16 +850,34 @@ static void armv8pmu_start(struct arm_pmu *cpu_pmu)
kvm_vcpu_pmu_resync_el0();
+ if (hw_events->branch_users)
+ brbe_enable(cpu_pmu);
+
/* Enable all counters */
armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
}
static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
{
+ struct pmu_hw_events *hw_events = this_cpu_ptr(cpu_pmu->hw_events);
+
+ if (hw_events->branch_users)
+ brbe_disable();
+
/* Disable all counters */
armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
}
+static void read_branch_records(struct pmu_hw_events *cpuc,
+ struct perf_event *event,
+ struct perf_sample_data *data)
+{
+ struct perf_branch_stack *branch_stack = cpuc->branch_stack;
+
+ brbe_read_filtered_entries(branch_stack, event);
+ perf_sample_save_brstack(data, event, branch_stack, NULL);
+}
+
static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
{
u64 pmovsr;
@@ -882,6 +928,9 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
if (!armpmu_event_set_period(event))
continue;
+ if (has_branch_stack(event))
+ read_branch_records(cpuc, event, &data);
+
/*
* Perf event overflow will queue the processing of the event as
* an irq_work which will be taken care of in the handling of
@@ -938,7 +987,7 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
/* Always prefer to place a cycle counter into the cycle counter. */
if ((evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) &&
- !armv8pmu_event_get_threshold(&event->attr)) {
+ !armv8pmu_event_get_threshold(&event->attr) && !has_branch_stack(event)) {
if (!test_and_set_bit(ARMV8_PMU_CYCLE_IDX, cpuc->used_mask))
return ARMV8_PMU_CYCLE_IDX;
else if (armv8pmu_event_is_64bit(event) &&
@@ -987,6 +1036,19 @@ static int armv8pmu_user_event_idx(struct perf_event *event)
return event->hw.idx + 1;
}
+static void armv8pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
+ struct task_struct *task, bool sched_in)
+{
+ struct arm_pmu *armpmu = *this_cpu_ptr(&cpu_armpmu);
+ struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);
+
+ if (!hw_events->branch_users)
+ return;
+
+ if (sched_in)
+ brbe_invalidate();
+}
+
/*
* Add an event filter to a given event.
*/
@@ -1004,6 +1066,13 @@ static int armv8pmu_set_event_filter(struct hw_perf_event *event,
return -EOPNOTSUPP;
}
+ if (has_branch_stack(perf_event)) {
+ if (!brbe_num_branch_records(cpu_pmu) || !brbe_branch_attr_valid(perf_event))
+ return -EOPNOTSUPP;
+
+ perf_event->attach_state |= PERF_ATTACH_SCHED_CB;
+ }
+
/*
* If we're running in hyp mode, then we *are* the hypervisor.
* Therefore we ignore exclude_hv in this configuration, since
@@ -1070,6 +1139,11 @@ static void armv8pmu_reset(void *info)
/* Clear the counters we flip at guest entry/exit */
kvm_clr_pmu_events(mask);
+ if (brbe_num_branch_records(cpu_pmu)) {
+ brbe_disable();
+ brbe_invalidate();
+ }
+
/*
* Initialize & Reset PMNC. Request overflow interrupt for
* 64 bit cycle counter but cheat in armv8pmu_write_counter().
@@ -1238,6 +1312,25 @@ static void __armv8pmu_probe_pmu(void *info)
cpu_pmu->reg_pmmir = read_pmmir();
else
cpu_pmu->reg_pmmir = 0;
+
+ brbe_probe(cpu_pmu);
+}
+
+static int branch_records_alloc(struct arm_pmu *armpmu)
+{
+ size_t size = struct_size_t(struct perf_branch_stack, entries,
+ brbe_num_branch_records(armpmu));
+ int cpu;
+
+ for_each_cpu(cpu, &armpmu->supported_cpus) {
+ struct pmu_hw_events *events_cpu;
+
+ events_cpu = per_cpu_ptr(armpmu->hw_events, cpu);
+ events_cpu->branch_stack = kmalloc(size, GFP_KERNEL);
+ if (!events_cpu->branch_stack)
+ return -ENOMEM;
+ }
+ return 0;
}
static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
@@ -1254,7 +1347,15 @@ static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
if (ret)
return ret;
- return probe.present ? 0 : -ENODEV;
+ if (!probe.present)
+ return -ENODEV;
+
+ if (brbe_num_branch_records(cpu_pmu)) {
+ ret = branch_records_alloc(cpu_pmu);
+ if (ret)
+ return ret;
+ }
+ return 0;
}
static void armv8pmu_disable_user_access_ipi(void *unused)
@@ -1313,6 +1414,8 @@ static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name,
cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
cpu_pmu->pmu.event_idx = armv8pmu_user_event_idx;
+ if (brbe_num_branch_records(cpu_pmu))
+ cpu_pmu->pmu.sched_task = armv8pmu_sched_task;
cpu_pmu->name = name;
cpu_pmu->map_event = map_event;
diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c
index 3efed8839a4e..369e77ad5f13 100644
--- a/drivers/perf/arm_spe_pmu.c
+++ b/drivers/perf/arm_spe_pmu.c
@@ -308,17 +308,21 @@ static u64 arm_spe_event_to_pmscr(struct perf_event *event)
static void arm_spe_event_sanitise_period(struct perf_event *event)
{
- struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
u64 period = event->hw.sample_period;
u64 max_period = PMSIRR_EL1_INTERVAL_MASK;
- if (period < spe_pmu->min_period)
- period = spe_pmu->min_period;
- else if (period > max_period)
- period = max_period;
- else
- period &= max_period;
+ /*
+ * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a
+ * recommendation for the minimum interval, not a hardware limitation.
+ *
+ * According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1,
+ * Sampling Interval Reload Register, the INTERVAL field (bits [31:8])
+ * states: "Software must set this to a nonzero value". Use 1 as the
+ * minimum value.
+ */
+ u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1);
+ period = clamp_t(u64, period, min_period, max_period) & max_period;
event->hw.sample_period = period;
}
diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c
index d6693519eaee..d094030220bf 100644
--- a/drivers/perf/cxl_pmu.c
+++ b/drivers/perf/cxl_pmu.c
@@ -113,7 +113,7 @@ struct cxl_pmu_info {
/*
* All CPMU counters are discoverable via the Event Capabilities Registers.
- * Each Event Capability register contains a a VID / GroupID.
+ * Each Event Capability register contains a VID / GroupID.
* A counter may then count any combination (by summing) of events in
* that group which are in the Supported Events Bitmask.
* However, there are some complexities to the scheme.
@@ -406,7 +406,7 @@ static struct attribute *cxl_pmu_event_attrs[] = {
CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)),
CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)),
CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)),
- /* CXL rev 3.1 Table 3-50 S2M NDR Opcopdes */
+ /* CXL rev 3.1 Table 3-50 S2M NDR Opcodes */
CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)),
CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)),
CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)),
@@ -627,7 +627,7 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags)
hwc->state = 0;
/*
- * Currently only hdm filter control is implemnted, this code will
+ * Currently only hdm filter control is implemented, this code will
* want generalizing when more filters are added.
*/
if (info->filter_hdm) {
@@ -834,8 +834,8 @@ static int cxl_pmu_probe(struct device *dev)
if (rc)
return rc;
- info->hw_events = devm_kcalloc(dev, sizeof(*info->hw_events),
- info->num_counters, GFP_KERNEL);
+ info->hw_events = devm_kcalloc(dev, info->num_counters,
+ sizeof(*info->hw_events), GFP_KERNEL);
if (!info->hw_events)
return -ENOMEM;
@@ -873,7 +873,7 @@ static int cxl_pmu_probe(struct device *dev)
return rc;
irq = rc;
- irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow\n", dev_name);
+ irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow", dev_name);
if (!irq_name)
return -ENOMEM;
diff --git a/drivers/perf/fsl_imx9_ddr_perf.c b/drivers/perf/fsl_imx9_ddr_perf.c
index 843f163e6c33..267754fdf581 100644
--- a/drivers/perf/fsl_imx9_ddr_perf.c
+++ b/drivers/perf/fsl_imx9_ddr_perf.c
@@ -461,9 +461,11 @@ static void imx93_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,
int counter, int axi_id, int axi_mask)
{
u32 pmcfg1, pmcfg2;
- u32 mask[] = { MX93_PMCFG1_RD_TRANS_FILT_EN,
- MX93_PMCFG1_WR_TRANS_FILT_EN,
- MX93_PMCFG1_RD_BT_FILT_EN };
+ static const u32 mask[] = {
+ MX93_PMCFG1_RD_TRANS_FILT_EN,
+ MX93_PMCFG1_WR_TRANS_FILT_EN,
+ MX93_PMCFG1_RD_BT_FILT_EN
+ };
pmcfg1 = readl_relaxed(pmu->base + PMCFG1);
diff --git a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c
index 7e490f8868f2..21c494881ca0 100644
--- a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c
@@ -43,12 +43,21 @@
#define DDRC_V2_EVENT_TYPE 0xe74
#define DDRC_V2_PERF_CTRL 0xeA0
+/* DDRC interrupt registers definition in v3 */
+#define DDRC_V3_INT_MASK 0x534
+#define DDRC_V3_INT_STATUS 0x538
+#define DDRC_V3_INT_CLEAR 0x53C
+
/* DDRC has 8-counters */
#define DDRC_NR_COUNTERS 0x8
#define DDRC_V1_PERF_CTRL_EN 0x2
#define DDRC_V2_PERF_CTRL_EN 0x1
#define DDRC_V1_NR_EVENTS 0x7
-#define DDRC_V2_NR_EVENTS 0x90
+#define DDRC_V2_NR_EVENTS 0xFF
+
+#define DDRC_EVENT_CNTn(base, n) ((base) + (n) * 8)
+#define DDRC_EVENT_TYPEn(base, n) ((base) + (n) * 4)
+#define DDRC_UNIMPLEMENTED_REG GENMASK(31, 0)
/*
* For PMU v1, there are eight-events and every event has been mapped
@@ -63,47 +72,37 @@ static const u32 ddrc_reg_off[] = {
DDRC_PRE_CMD, DDRC_ACT_CMD, DDRC_RNK_CHG, DDRC_RW_CHG
};
-/*
- * Select the counter register offset using the counter index.
- * In PMU v1, there are no programmable counter, the count
- * is read form the statistics counter register itself.
- */
-static u32 hisi_ddrc_pmu_v1_get_counter_offset(int cntr_idx)
-{
- return ddrc_reg_off[cntr_idx];
-}
-
-static u32 hisi_ddrc_pmu_v2_get_counter_offset(int cntr_idx)
-{
- return DDRC_V2_EVENT_CNT + cntr_idx * 8;
-}
+struct hisi_ddrc_pmu_regs {
+ u32 event_cnt;
+ u32 event_ctrl;
+ u32 event_type;
+ u32 perf_ctrl;
+ u32 perf_ctrl_en;
+ u32 int_mask;
+ u32 int_clear;
+ u32 int_status;
+};
-static u64 hisi_ddrc_pmu_v1_read_counter(struct hisi_pmu *ddrc_pmu,
+static u64 hisi_ddrc_pmu_read_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
- return readl(ddrc_pmu->base +
- hisi_ddrc_pmu_v1_get_counter_offset(hwc->idx));
-}
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
-static void hisi_ddrc_pmu_v1_write_counter(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc, u64 val)
-{
- writel((u32)val,
- ddrc_pmu->base + hisi_ddrc_pmu_v1_get_counter_offset(hwc->idx));
-}
+ if (regs->event_cnt == DDRC_UNIMPLEMENTED_REG)
+ return readl(ddrc_pmu->base + ddrc_reg_off[hwc->idx]);
-static u64 hisi_ddrc_pmu_v2_read_counter(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc)
-{
- return readq(ddrc_pmu->base +
- hisi_ddrc_pmu_v2_get_counter_offset(hwc->idx));
+ return readq(ddrc_pmu->base + DDRC_EVENT_CNTn(regs->event_cnt, hwc->idx));
}
-static void hisi_ddrc_pmu_v2_write_counter(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc, u64 val)
+static void hisi_ddrc_pmu_write_counter(struct hisi_pmu *ddrc_pmu,
+ struct hw_perf_event *hwc, u64 val)
{
- writeq(val,
- ddrc_pmu->base + hisi_ddrc_pmu_v2_get_counter_offset(hwc->idx));
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
+
+ if (regs->event_cnt == DDRC_UNIMPLEMENTED_REG)
+ writel((u32)val, ddrc_pmu->base + ddrc_reg_off[hwc->idx]);
+ else
+ writeq(val, ddrc_pmu->base + DDRC_EVENT_CNTn(regs->event_cnt, hwc->idx));
}
/*
@@ -114,54 +113,12 @@ static void hisi_ddrc_pmu_v2_write_counter(struct hisi_pmu *ddrc_pmu,
static void hisi_ddrc_pmu_write_evtype(struct hisi_pmu *ddrc_pmu, int idx,
u32 type)
{
- u32 offset;
-
- if (ddrc_pmu->identifier >= HISI_PMU_V2) {
- offset = DDRC_V2_EVENT_TYPE + 4 * idx;
- writel(type, ddrc_pmu->base + offset);
- }
-}
-
-static void hisi_ddrc_pmu_v1_start_counters(struct hisi_pmu *ddrc_pmu)
-{
- u32 val;
-
- /* Set perf_enable in DDRC_PERF_CTRL to start event counting */
- val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
- val |= DDRC_V1_PERF_CTRL_EN;
- writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
-}
-
-static void hisi_ddrc_pmu_v1_stop_counters(struct hisi_pmu *ddrc_pmu)
-{
- u32 val;
-
- /* Clear perf_enable in DDRC_PERF_CTRL to stop event counting */
- val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
- val &= ~DDRC_V1_PERF_CTRL_EN;
- writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
-}
-
-static void hisi_ddrc_pmu_v1_enable_counter(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc)
-{
- u32 val;
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
- /* Set counter index(event code) in DDRC_EVENT_CTRL register */
- val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL);
- val |= (1 << GET_DDRC_EVENTID(hwc));
- writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
-}
+ if (regs->event_type == DDRC_UNIMPLEMENTED_REG)
+ return;
-static void hisi_ddrc_pmu_v1_disable_counter(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc)
-{
- u32 val;
-
- /* Clear counter index(event code) in DDRC_EVENT_CTRL register */
- val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL);
- val &= ~(1 << GET_DDRC_EVENTID(hwc));
- writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
+ writel(type, ddrc_pmu->base + DDRC_EVENT_TYPEn(regs->event_type, idx));
}
static int hisi_ddrc_pmu_v1_get_event_idx(struct perf_event *event)
@@ -180,120 +137,96 @@ static int hisi_ddrc_pmu_v1_get_event_idx(struct perf_event *event)
return idx;
}
-static int hisi_ddrc_pmu_v2_get_event_idx(struct perf_event *event)
+static int hisi_ddrc_pmu_get_event_idx(struct perf_event *event)
{
+ struct hisi_pmu *ddrc_pmu = to_hisi_pmu(event->pmu);
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
+
+ if (regs->event_type == DDRC_UNIMPLEMENTED_REG)
+ return hisi_ddrc_pmu_v1_get_event_idx(event);
+
return hisi_uncore_pmu_get_event_idx(event);
}
-static void hisi_ddrc_pmu_v2_start_counters(struct hisi_pmu *ddrc_pmu)
+static void hisi_ddrc_pmu_start_counters(struct hisi_pmu *ddrc_pmu)
{
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
u32 val;
- val = readl(ddrc_pmu->base + DDRC_V2_PERF_CTRL);
- val |= DDRC_V2_PERF_CTRL_EN;
- writel(val, ddrc_pmu->base + DDRC_V2_PERF_CTRL);
+ val = readl(ddrc_pmu->base + regs->perf_ctrl);
+ val |= regs->perf_ctrl_en;
+ writel(val, ddrc_pmu->base + regs->perf_ctrl);
}
-static void hisi_ddrc_pmu_v2_stop_counters(struct hisi_pmu *ddrc_pmu)
+static void hisi_ddrc_pmu_stop_counters(struct hisi_pmu *ddrc_pmu)
{
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
u32 val;
- val = readl(ddrc_pmu->base + DDRC_V2_PERF_CTRL);
- val &= ~DDRC_V2_PERF_CTRL_EN;
- writel(val, ddrc_pmu->base + DDRC_V2_PERF_CTRL);
+ val = readl(ddrc_pmu->base + regs->perf_ctrl);
+ val &= ~regs->perf_ctrl_en;
+ writel(val, ddrc_pmu->base + regs->perf_ctrl);
}
-static void hisi_ddrc_pmu_v2_enable_counter(struct hisi_pmu *ddrc_pmu,
+static void hisi_ddrc_pmu_enable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
u32 val;
- val = readl(ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
- val |= 1 << hwc->idx;
- writel(val, ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
+ val = readl(ddrc_pmu->base + regs->event_ctrl);
+ val |= BIT_ULL(hwc->idx);
+ writel(val, ddrc_pmu->base + regs->event_ctrl);
}
-static void hisi_ddrc_pmu_v2_disable_counter(struct hisi_pmu *ddrc_pmu,
+static void hisi_ddrc_pmu_disable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
u32 val;
- val = readl(ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
- val &= ~(1 << hwc->idx);
- writel(val, ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
+ val = readl(ddrc_pmu->base + regs->event_ctrl);
+ val &= ~BIT_ULL(hwc->idx);
+ writel(val, ddrc_pmu->base + regs->event_ctrl);
}
-static void hisi_ddrc_pmu_v1_enable_counter_int(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc)
-{
- u32 val;
-
- /* Write 0 to enable interrupt */
- val = readl(ddrc_pmu->base + DDRC_INT_MASK);
- val &= ~(1 << hwc->idx);
- writel(val, ddrc_pmu->base + DDRC_INT_MASK);
-}
-
-static void hisi_ddrc_pmu_v1_disable_counter_int(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc)
-{
- u32 val;
-
- /* Write 1 to mask interrupt */
- val = readl(ddrc_pmu->base + DDRC_INT_MASK);
- val |= 1 << hwc->idx;
- writel(val, ddrc_pmu->base + DDRC_INT_MASK);
-}
-
-static void hisi_ddrc_pmu_v2_enable_counter_int(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc)
+static void hisi_ddrc_pmu_enable_counter_int(struct hisi_pmu *ddrc_pmu,
+ struct hw_perf_event *hwc)
{
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
u32 val;
- val = readl(ddrc_pmu->base + DDRC_V2_INT_MASK);
- val &= ~(1 << hwc->idx);
- writel(val, ddrc_pmu->base + DDRC_V2_INT_MASK);
+ val = readl(ddrc_pmu->base + regs->int_mask);
+ val &= ~BIT_ULL(hwc->idx);
+ writel(val, ddrc_pmu->base + regs->int_mask);
}
-static void hisi_ddrc_pmu_v2_disable_counter_int(struct hisi_pmu *ddrc_pmu,
- struct hw_perf_event *hwc)
+static void hisi_ddrc_pmu_disable_counter_int(struct hisi_pmu *ddrc_pmu,
+ struct hw_perf_event *hwc)
{
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
u32 val;
- val = readl(ddrc_pmu->base + DDRC_V2_INT_MASK);
- val |= 1 << hwc->idx;
- writel(val, ddrc_pmu->base + DDRC_V2_INT_MASK);
+ val = readl(ddrc_pmu->base + regs->int_mask);
+ val |= BIT_ULL(hwc->idx);
+ writel(val, ddrc_pmu->base + regs->int_mask);
}
-static u32 hisi_ddrc_pmu_v1_get_int_status(struct hisi_pmu *ddrc_pmu)
+static u32 hisi_ddrc_pmu_get_int_status(struct hisi_pmu *ddrc_pmu)
{
- return readl(ddrc_pmu->base + DDRC_INT_STATUS);
-}
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
-static void hisi_ddrc_pmu_v1_clear_int_status(struct hisi_pmu *ddrc_pmu,
- int idx)
-{
- writel(1 << idx, ddrc_pmu->base + DDRC_INT_CLEAR);
+ return readl(ddrc_pmu->base + regs->int_status);
}
-static u32 hisi_ddrc_pmu_v2_get_int_status(struct hisi_pmu *ddrc_pmu)
+static void hisi_ddrc_pmu_clear_int_status(struct hisi_pmu *ddrc_pmu,
+ int idx)
{
- return readl(ddrc_pmu->base + DDRC_V2_INT_STATUS);
-}
+ struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private;
-static void hisi_ddrc_pmu_v2_clear_int_status(struct hisi_pmu *ddrc_pmu,
- int idx)
-{
- writel(1 << idx, ddrc_pmu->base + DDRC_V2_INT_CLEAR);
+ writel(1 << idx, ddrc_pmu->base + regs->int_clear);
}
-static const struct acpi_device_id hisi_ddrc_pmu_acpi_match[] = {
- { "HISI0233", },
- { "HISI0234", },
- {}
-};
-MODULE_DEVICE_TABLE(acpi, hisi_ddrc_pmu_acpi_match);
-
static int hisi_ddrc_pmu_init_data(struct platform_device *pdev,
struct hisi_pmu *ddrc_pmu)
{
@@ -314,6 +247,10 @@ static int hisi_ddrc_pmu_init_data(struct platform_device *pdev,
return -EINVAL;
}
+ ddrc_pmu->dev_info = device_get_match_data(&pdev->dev);
+ if (!ddrc_pmu->dev_info)
+ return -ENODEV;
+
ddrc_pmu->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(ddrc_pmu->base)) {
dev_err(&pdev->dev, "ioremap failed for ddrc_pmu resource\n");
@@ -396,34 +333,19 @@ static const struct attribute_group *hisi_ddrc_pmu_v2_attr_groups[] = {
NULL
};
-static const struct hisi_uncore_ops hisi_uncore_ddrc_v1_ops = {
- .write_evtype = hisi_ddrc_pmu_write_evtype,
- .get_event_idx = hisi_ddrc_pmu_v1_get_event_idx,
- .start_counters = hisi_ddrc_pmu_v1_start_counters,
- .stop_counters = hisi_ddrc_pmu_v1_stop_counters,
- .enable_counter = hisi_ddrc_pmu_v1_enable_counter,
- .disable_counter = hisi_ddrc_pmu_v1_disable_counter,
- .enable_counter_int = hisi_ddrc_pmu_v1_enable_counter_int,
- .disable_counter_int = hisi_ddrc_pmu_v1_disable_counter_int,
- .write_counter = hisi_ddrc_pmu_v1_write_counter,
- .read_counter = hisi_ddrc_pmu_v1_read_counter,
- .get_int_status = hisi_ddrc_pmu_v1_get_int_status,
- .clear_int_status = hisi_ddrc_pmu_v1_clear_int_status,
-};
-
-static const struct hisi_uncore_ops hisi_uncore_ddrc_v2_ops = {
+static const struct hisi_uncore_ops hisi_uncore_ddrc_ops = {
.write_evtype = hisi_ddrc_pmu_write_evtype,
- .get_event_idx = hisi_ddrc_pmu_v2_get_event_idx,
- .start_counters = hisi_ddrc_pmu_v2_start_counters,
- .stop_counters = hisi_ddrc_pmu_v2_stop_counters,
- .enable_counter = hisi_ddrc_pmu_v2_enable_counter,
- .disable_counter = hisi_ddrc_pmu_v2_disable_counter,
- .enable_counter_int = hisi_ddrc_pmu_v2_enable_counter_int,
- .disable_counter_int = hisi_ddrc_pmu_v2_disable_counter_int,
- .write_counter = hisi_ddrc_pmu_v2_write_counter,
- .read_counter = hisi_ddrc_pmu_v2_read_counter,
- .get_int_status = hisi_ddrc_pmu_v2_get_int_status,
- .clear_int_status = hisi_ddrc_pmu_v2_clear_int_status,
+ .get_event_idx = hisi_ddrc_pmu_get_event_idx,
+ .start_counters = hisi_ddrc_pmu_start_counters,
+ .stop_counters = hisi_ddrc_pmu_stop_counters,
+ .enable_counter = hisi_ddrc_pmu_enable_counter,
+ .disable_counter = hisi_ddrc_pmu_disable_counter,
+ .enable_counter_int = hisi_ddrc_pmu_enable_counter_int,
+ .disable_counter_int = hisi_ddrc_pmu_disable_counter_int,
+ .write_counter = hisi_ddrc_pmu_write_counter,
+ .read_counter = hisi_ddrc_pmu_read_counter,
+ .get_int_status = hisi_ddrc_pmu_get_int_status,
+ .clear_int_status = hisi_ddrc_pmu_clear_int_status,
};
static int hisi_ddrc_pmu_dev_probe(struct platform_device *pdev,
@@ -439,18 +361,10 @@ static int hisi_ddrc_pmu_dev_probe(struct platform_device *pdev,
if (ret)
return ret;
- if (ddrc_pmu->identifier >= HISI_PMU_V2) {
- ddrc_pmu->counter_bits = 48;
- ddrc_pmu->check_event = DDRC_V2_NR_EVENTS;
- ddrc_pmu->pmu_events.attr_groups = hisi_ddrc_pmu_v2_attr_groups;
- ddrc_pmu->ops = &hisi_uncore_ddrc_v2_ops;
- } else {
- ddrc_pmu->counter_bits = 32;
- ddrc_pmu->check_event = DDRC_V1_NR_EVENTS;
- ddrc_pmu->pmu_events.attr_groups = hisi_ddrc_pmu_v1_attr_groups;
- ddrc_pmu->ops = &hisi_uncore_ddrc_v1_ops;
- }
-
+ ddrc_pmu->pmu_events.attr_groups = ddrc_pmu->dev_info->attr_groups;
+ ddrc_pmu->counter_bits = ddrc_pmu->dev_info->counter_bits;
+ ddrc_pmu->check_event = ddrc_pmu->dev_info->check_event;
+ ddrc_pmu->ops = &hisi_uncore_ddrc_ops;
ddrc_pmu->num_counters = DDRC_NR_COUNTERS;
ddrc_pmu->dev = &pdev->dev;
ddrc_pmu->on_cpu = -1;
@@ -515,6 +429,68 @@ static void hisi_ddrc_pmu_remove(struct platform_device *pdev)
&ddrc_pmu->node);
}
+static struct hisi_ddrc_pmu_regs hisi_ddrc_v1_pmu_regs = {
+ .event_cnt = DDRC_UNIMPLEMENTED_REG,
+ .event_ctrl = DDRC_EVENT_CTRL,
+ .event_type = DDRC_UNIMPLEMENTED_REG,
+ .perf_ctrl = DDRC_PERF_CTRL,
+ .perf_ctrl_en = DDRC_V1_PERF_CTRL_EN,
+ .int_mask = DDRC_INT_MASK,
+ .int_clear = DDRC_INT_CLEAR,
+ .int_status = DDRC_INT_STATUS,
+};
+
+static const struct hisi_pmu_dev_info hisi_ddrc_v1 = {
+ .counter_bits = 32,
+ .check_event = DDRC_V1_NR_EVENTS,
+ .attr_groups = hisi_ddrc_pmu_v1_attr_groups,
+ .private = &hisi_ddrc_v1_pmu_regs,
+};
+
+static struct hisi_ddrc_pmu_regs hisi_ddrc_v2_pmu_regs = {
+ .event_cnt = DDRC_V2_EVENT_CNT,
+ .event_ctrl = DDRC_V2_EVENT_CTRL,
+ .event_type = DDRC_V2_EVENT_TYPE,
+ .perf_ctrl = DDRC_V2_PERF_CTRL,
+ .perf_ctrl_en = DDRC_V2_PERF_CTRL_EN,
+ .int_mask = DDRC_V2_INT_MASK,
+ .int_clear = DDRC_V2_INT_CLEAR,
+ .int_status = DDRC_V2_INT_STATUS,
+};
+
+static const struct hisi_pmu_dev_info hisi_ddrc_v2 = {
+ .counter_bits = 48,
+ .check_event = DDRC_V2_NR_EVENTS,
+ .attr_groups = hisi_ddrc_pmu_v2_attr_groups,
+ .private = &hisi_ddrc_v2_pmu_regs,
+};
+
+static struct hisi_ddrc_pmu_regs hisi_ddrc_v3_pmu_regs = {
+ .event_cnt = DDRC_V2_EVENT_CNT,
+ .event_ctrl = DDRC_V2_EVENT_CTRL,
+ .event_type = DDRC_V2_EVENT_TYPE,
+ .perf_ctrl = DDRC_V2_PERF_CTRL,
+ .perf_ctrl_en = DDRC_V2_PERF_CTRL_EN,
+ .int_mask = DDRC_V3_INT_MASK,
+ .int_clear = DDRC_V3_INT_CLEAR,
+ .int_status = DDRC_V3_INT_STATUS,
+};
+
+static const struct hisi_pmu_dev_info hisi_ddrc_v3 = {
+ .counter_bits = 48,
+ .check_event = DDRC_V2_NR_EVENTS,
+ .attr_groups = hisi_ddrc_pmu_v2_attr_groups,
+ .private = &hisi_ddrc_v3_pmu_regs,
+};
+
+static const struct acpi_device_id hisi_ddrc_pmu_acpi_match[] = {
+ { "HISI0233", (kernel_ulong_t)&hisi_ddrc_v1 },
+ { "HISI0234", (kernel_ulong_t)&hisi_ddrc_v2 },
+ { "HISI0235", (kernel_ulong_t)&hisi_ddrc_v3 },
+ {}
+};
+MODULE_DEVICE_TABLE(acpi, hisi_ddrc_pmu_acpi_match);
+
static struct platform_driver hisi_ddrc_pmu_driver = {
.driver = {
.name = "hisi_ddrc_pmu",
diff --git a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c
index ca609db86046..97cfaa586a87 100644
--- a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c
@@ -47,9 +47,9 @@
#define HHA_SRCID_CMD GENMASK(16, 6)
#define HHA_SRCID_MSK GENMASK(30, 20)
#define HHA_DATSRC_SKT_EN BIT(23)
-#define HHA_EVTYPE_NONE 0xff
+#define HHA_EVTYPE_MASK GENMASK(7, 0)
#define HHA_V1_NR_EVENT 0x65
-#define HHA_V2_NR_EVENT 0xCE
+#define HHA_V2_NR_EVENT 0xFF
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_cmd, config1, 10, 0);
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_msk, config1, 21, 11);
@@ -197,7 +197,7 @@ static void hisi_hha_pmu_write_evtype(struct hisi_pmu *hha_pmu, int idx,
/* Write event code to HHA_EVENT_TYPEx register */
val = readl(hha_pmu->base + reg);
- val &= ~(HHA_EVTYPE_NONE << shift);
+ val &= ~(HHA_EVTYPE_MASK << shift);
val |= (type << shift);
writel(val, hha_pmu->base + reg);
}
diff --git a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c
index a0142684e379..80108c63cb60 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c
@@ -440,7 +440,7 @@ static int hisi_pa_pmu_dev_probe(struct platform_device *pdev,
pa_pmu->pmu_events.attr_groups = pa_pmu->dev_info->attr_groups;
pa_pmu->num_counters = PA_NR_COUNTERS;
pa_pmu->ops = &hisi_uncore_pa_ops;
- pa_pmu->check_event = 0xB0;
+ pa_pmu->check_event = PA_EVTYPE_MASK;
pa_pmu->counter_bits = 64;
pa_pmu->dev = &pdev->dev;
pa_pmu->on_cpu = -1;
diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c
index ef058b1dd509..a449651f79c9 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c
@@ -510,7 +510,9 @@ int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
return 0;
hisi_pmu->on_cpu = cpumask_local_spread(0, dev_to_node(hisi_pmu->dev));
- WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(hisi_pmu->on_cpu)));
+ if (hisi_pmu->irq > 0)
+ WARN_ON(irq_set_affinity(hisi_pmu->irq,
+ cpumask_of(hisi_pmu->on_cpu)));
return 0;
}
@@ -525,7 +527,8 @@ int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
hisi_pmu->on_cpu = cpu;
/* Overflow interrupt also should use the same CPU */
- WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
+ if (hisi_pmu->irq > 0)
+ WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
return 0;
}
@@ -560,7 +563,9 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
/* Use this CPU for event counting */
hisi_pmu->on_cpu = target;
- WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
+
+ if (hisi_pmu->irq > 0)
+ WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
return 0;
}
diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.h b/drivers/perf/hisilicon/hisi_uncore_pmu.h
index f4fed2544877..777675838b80 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pmu.h
+++ b/drivers/perf/hisilicon/hisi_uncore_pmu.h
@@ -72,6 +72,8 @@ struct hisi_uncore_ops {
struct hisi_pmu_dev_info {
const char *name;
const struct attribute_group **attr_groups;
+ u32 counter_bits;
+ u32 check_event;
void *private;
};
diff --git a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c
index dbd079016fc4..cd32d606df05 100644
--- a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c
@@ -28,6 +28,18 @@
#define SLLC_VERSION 0x1cf0
#define SLLC_EVENT_CNT0_L 0x1d00
+/* SLLC registers definition in v3 */
+#define SLLC_V3_INT_MASK 0x6834
+#define SLLC_V3_INT_STATUS 0x6838
+#define SLLC_V3_INT_CLEAR 0x683c
+#define SLLC_V3_VERSION 0x6c00
+#define SLLC_V3_PERF_CTRL 0x6d00
+#define SLLC_V3_SRCID_CTRL 0x6d04
+#define SLLC_V3_TGTID_CTRL 0x6d08
+#define SLLC_V3_EVENT_CTRL 0x6d14
+#define SLLC_V3_EVENT_TYPE0 0x6d18
+#define SLLC_V3_EVENT_CNT0_L 0x6e00
+
#define SLLC_EVTYPE_MASK 0xff
#define SLLC_PERF_CTRL_EN BIT(0)
#define SLLC_FILT_EN BIT(1)
@@ -40,7 +52,14 @@
#define SLLC_TGTID_MAX_SHIFT 12
#define SLLC_SRCID_CMD_SHIFT 1
#define SLLC_SRCID_MSK_SHIFT 12
-#define SLLC_NR_EVENTS 0x80
+
+#define SLLC_V3_TGTID_MIN_SHIFT 1
+#define SLLC_V3_TGTID_MAX_SHIFT 10
+#define SLLC_V3_SRCID_CMD_SHIFT 1
+#define SLLC_V3_SRCID_MSK_SHIFT 10
+
+#define SLLC_NR_EVENTS 0xff
+#define SLLC_EVENT_CNTn(cnt0, n) ((cnt0) + (n) * 8)
HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_min, config1, 10, 0);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_max, config1, 21, 11);
@@ -48,6 +67,23 @@ HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_cmd, config1, 32, 22);
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_msk, config1, 43, 33);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tracetag_en, config1, 44, 44);
+struct hisi_sllc_pmu_regs {
+ u32 int_mask;
+ u32 int_clear;
+ u32 int_status;
+ u32 perf_ctrl;
+ u32 srcid_ctrl;
+ u32 srcid_cmd_shift;
+ u32 srcid_mask_shift;
+ u32 tgtid_ctrl;
+ u32 tgtid_min_shift;
+ u32 tgtid_max_shift;
+ u32 event_ctrl;
+ u32 event_type0;
+ u32 version;
+ u32 event_cnt0;
+};
+
static bool tgtid_is_valid(u32 max, u32 min)
{
return max > 0 && max >= min;
@@ -56,96 +92,104 @@ static bool tgtid_is_valid(u32 max, u32 min)
static void hisi_sllc_pmu_enable_tracetag(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 tt_en = hisi_get_tracetag_en(event);
if (tt_en) {
u32 val;
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val |= SLLC_TRACETAG_EN | SLLC_FILT_EN;
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
}
static void hisi_sllc_pmu_disable_tracetag(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 tt_en = hisi_get_tracetag_en(event);
if (tt_en) {
u32 val;
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val &= ~(SLLC_TRACETAG_EN | SLLC_FILT_EN);
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
}
static void hisi_sllc_pmu_config_tgtid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 min = hisi_get_tgtid_min(event);
u32 max = hisi_get_tgtid_max(event);
if (tgtid_is_valid(max, min)) {
- u32 val = (max << SLLC_TGTID_MAX_SHIFT) | (min << SLLC_TGTID_MIN_SHIFT);
+ u32 val = (max << regs->tgtid_max_shift) |
+ (min << regs->tgtid_min_shift);
- writel(val, sllc_pmu->base + SLLC_TGTID_CTRL);
+ writel(val, sllc_pmu->base + regs->tgtid_ctrl);
/* Enable the tgtid */
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val |= SLLC_TGTID_EN | SLLC_FILT_EN;
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
}
static void hisi_sllc_pmu_clear_tgtid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 min = hisi_get_tgtid_min(event);
u32 max = hisi_get_tgtid_max(event);
if (tgtid_is_valid(max, min)) {
u32 val;
- writel(SLLC_TGTID_NONE, sllc_pmu->base + SLLC_TGTID_CTRL);
+ writel(SLLC_TGTID_NONE, sllc_pmu->base + regs->tgtid_ctrl);
/* Disable the tgtid */
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val &= ~(SLLC_TGTID_EN | SLLC_FILT_EN);
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
}
static void hisi_sllc_pmu_config_srcid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd) {
u32 val, msk;
msk = hisi_get_srcid_msk(event);
- val = (cmd << SLLC_SRCID_CMD_SHIFT) | (msk << SLLC_SRCID_MSK_SHIFT);
- writel(val, sllc_pmu->base + SLLC_SRCID_CTRL);
+ val = (cmd << regs->srcid_cmd_shift) |
+ (msk << regs->srcid_mask_shift);
+ writel(val, sllc_pmu->base + regs->srcid_ctrl);
/* Enable the srcid */
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val |= SLLC_SRCID_EN | SLLC_FILT_EN;
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
}
static void hisi_sllc_pmu_clear_srcid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd) {
u32 val;
- writel(SLLC_SRCID_NONE, sllc_pmu->base + SLLC_SRCID_CTRL);
+ writel(SLLC_SRCID_NONE, sllc_pmu->base + regs->srcid_ctrl);
/* Disable the srcid */
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val &= ~(SLLC_SRCID_EN | SLLC_FILT_EN);
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
}
@@ -167,29 +211,27 @@ static void hisi_sllc_pmu_clear_filter(struct perf_event *event)
}
}
-static u32 hisi_sllc_pmu_get_counter_offset(int idx)
-{
- return (SLLC_EVENT_CNT0_L + idx * 8);
-}
-
static u64 hisi_sllc_pmu_read_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
- return readq(sllc_pmu->base +
- hisi_sllc_pmu_get_counter_offset(hwc->idx));
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
+
+ return readq(sllc_pmu->base + SLLC_EVENT_CNTn(regs->event_cnt0, hwc->idx));
}
static void hisi_sllc_pmu_write_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc, u64 val)
{
- writeq(val, sllc_pmu->base +
- hisi_sllc_pmu_get_counter_offset(hwc->idx));
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
+
+ writeq(val, sllc_pmu->base + SLLC_EVENT_CNTn(regs->event_cnt0, hwc->idx));
}
static void hisi_sllc_pmu_write_evtype(struct hisi_pmu *sllc_pmu, int idx,
u32 type)
{
- u32 reg, reg_idx, shift, val;
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
+ u32 reg, val;
/*
* Select the appropriate event select register(SLLC_EVENT_TYPE0/1).
@@ -198,96 +240,98 @@ static void hisi_sllc_pmu_write_evtype(struct hisi_pmu *sllc_pmu, int idx,
* SLLC_EVENT_TYPE0 is chosen. For the latter 4 hardware counters,
* SLLC_EVENT_TYPE1 is chosen.
*/
- reg = SLLC_EVENT_TYPE0 + (idx / 4) * 4;
- reg_idx = idx % 4;
- shift = 8 * reg_idx;
+ reg = regs->event_type0 + (idx / 4) * 4;
/* Write event code to SLLC_EVENT_TYPEx Register */
val = readl(sllc_pmu->base + reg);
- val &= ~(SLLC_EVTYPE_MASK << shift);
- val |= (type << shift);
+ val &= ~(SLLC_EVTYPE_MASK << HISI_PMU_EVTYPE_SHIFT(idx));
+ val |= (type << HISI_PMU_EVTYPE_SHIFT(idx));
writel(val, sllc_pmu->base + reg);
}
static void hisi_sllc_pmu_start_counters(struct hisi_pmu *sllc_pmu)
{
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 val;
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val |= SLLC_PERF_CTRL_EN;
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
static void hisi_sllc_pmu_stop_counters(struct hisi_pmu *sllc_pmu)
{
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 val;
- val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
+ val = readl(sllc_pmu->base + regs->perf_ctrl);
val &= ~(SLLC_PERF_CTRL_EN);
- writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
+ writel(val, sllc_pmu->base + regs->perf_ctrl);
}
static void hisi_sllc_pmu_enable_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 val;
- val = readl(sllc_pmu->base + SLLC_EVENT_CTRL);
- val |= 1 << hwc->idx;
- writel(val, sllc_pmu->base + SLLC_EVENT_CTRL);
+ val = readl(sllc_pmu->base + regs->event_ctrl);
+ val |= BIT_ULL(hwc->idx);
+ writel(val, sllc_pmu->base + regs->event_ctrl);
}
static void hisi_sllc_pmu_disable_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 val;
- val = readl(sllc_pmu->base + SLLC_EVENT_CTRL);
- val &= ~(1 << hwc->idx);
- writel(val, sllc_pmu->base + SLLC_EVENT_CTRL);
+ val = readl(sllc_pmu->base + regs->event_ctrl);
+ val &= ~BIT_ULL(hwc->idx);
+ writel(val, sllc_pmu->base + regs->event_ctrl);
}
static void hisi_sllc_pmu_enable_counter_int(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 val;
- val = readl(sllc_pmu->base + SLLC_INT_MASK);
- /* Write 0 to enable interrupt */
- val &= ~(1 << hwc->idx);
- writel(val, sllc_pmu->base + SLLC_INT_MASK);
+ val = readl(sllc_pmu->base + regs->int_mask);
+ val &= ~BIT_ULL(hwc->idx);
+ writel(val, sllc_pmu->base + regs->int_mask);
}
static void hisi_sllc_pmu_disable_counter_int(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
u32 val;
- val = readl(sllc_pmu->base + SLLC_INT_MASK);
- /* Write 1 to mask interrupt */
- val |= 1 << hwc->idx;
- writel(val, sllc_pmu->base + SLLC_INT_MASK);
+ val = readl(sllc_pmu->base + regs->int_mask);
+ val |= BIT_ULL(hwc->idx);
+ writel(val, sllc_pmu->base + regs->int_mask);
}
static u32 hisi_sllc_pmu_get_int_status(struct hisi_pmu *sllc_pmu)
{
- return readl(sllc_pmu->base + SLLC_INT_STATUS);
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
+
+ return readl(sllc_pmu->base + regs->int_status);
}
static void hisi_sllc_pmu_clear_int_status(struct hisi_pmu *sllc_pmu, int idx)
{
- writel(1 << idx, sllc_pmu->base + SLLC_INT_CLEAR);
-}
+ struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private;
-static const struct acpi_device_id hisi_sllc_pmu_acpi_match[] = {
- { "HISI0263", },
- {}
-};
-MODULE_DEVICE_TABLE(acpi, hisi_sllc_pmu_acpi_match);
+ writel(BIT_ULL(idx), sllc_pmu->base + regs->int_clear);
+}
static int hisi_sllc_pmu_init_data(struct platform_device *pdev,
struct hisi_pmu *sllc_pmu)
{
+ struct hisi_sllc_pmu_regs *regs;
+
hisi_uncore_pmu_init_topology(sllc_pmu, &pdev->dev);
/*
@@ -304,13 +348,18 @@ static int hisi_sllc_pmu_init_data(struct platform_device *pdev,
return -EINVAL;
}
+ sllc_pmu->dev_info = device_get_match_data(&pdev->dev);
+ if (!sllc_pmu->dev_info)
+ return -ENODEV;
+
sllc_pmu->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(sllc_pmu->base)) {
dev_err(&pdev->dev, "ioremap failed for sllc_pmu resource.\n");
return PTR_ERR(sllc_pmu->base);
}
- sllc_pmu->identifier = readl(sllc_pmu->base + SLLC_VERSION);
+ regs = sllc_pmu->dev_info->private;
+ sllc_pmu->identifier = readl(sllc_pmu->base + regs->version);
return 0;
}
@@ -352,6 +401,48 @@ static const struct attribute_group *hisi_sllc_pmu_v2_attr_groups[] = {
NULL
};
+static struct hisi_sllc_pmu_regs hisi_sllc_v2_pmu_regs = {
+ .int_mask = SLLC_INT_MASK,
+ .int_clear = SLLC_INT_CLEAR,
+ .int_status = SLLC_INT_STATUS,
+ .perf_ctrl = SLLC_PERF_CTRL,
+ .srcid_ctrl = SLLC_SRCID_CTRL,
+ .srcid_cmd_shift = SLLC_SRCID_CMD_SHIFT,
+ .srcid_mask_shift = SLLC_SRCID_MSK_SHIFT,
+ .tgtid_ctrl = SLLC_TGTID_CTRL,
+ .tgtid_min_shift = SLLC_TGTID_MIN_SHIFT,
+ .tgtid_max_shift = SLLC_TGTID_MAX_SHIFT,
+ .event_ctrl = SLLC_EVENT_CTRL,
+ .event_type0 = SLLC_EVENT_TYPE0,
+ .version = SLLC_VERSION,
+ .event_cnt0 = SLLC_EVENT_CNT0_L,
+};
+
+static const struct hisi_pmu_dev_info hisi_sllc_v2 = {
+ .private = &hisi_sllc_v2_pmu_regs,
+};
+
+static struct hisi_sllc_pmu_regs hisi_sllc_v3_pmu_regs = {
+ .int_mask = SLLC_V3_INT_MASK,
+ .int_clear = SLLC_V3_INT_CLEAR,
+ .int_status = SLLC_V3_INT_STATUS,
+ .perf_ctrl = SLLC_V3_PERF_CTRL,
+ .srcid_ctrl = SLLC_V3_SRCID_CTRL,
+ .srcid_cmd_shift = SLLC_V3_SRCID_CMD_SHIFT,
+ .srcid_mask_shift = SLLC_V3_SRCID_MSK_SHIFT,
+ .tgtid_ctrl = SLLC_V3_TGTID_CTRL,
+ .tgtid_min_shift = SLLC_V3_TGTID_MIN_SHIFT,
+ .tgtid_max_shift = SLLC_V3_TGTID_MAX_SHIFT,
+ .event_ctrl = SLLC_V3_EVENT_CTRL,
+ .event_type0 = SLLC_V3_EVENT_TYPE0,
+ .version = SLLC_V3_VERSION,
+ .event_cnt0 = SLLC_V3_EVENT_CNT0_L,
+};
+
+static const struct hisi_pmu_dev_info hisi_sllc_v3 = {
+ .private = &hisi_sllc_v3_pmu_regs,
+};
+
static const struct hisi_uncore_ops hisi_uncore_sllc_ops = {
.write_evtype = hisi_sllc_pmu_write_evtype,
.get_event_idx = hisi_uncore_pmu_get_event_idx,
@@ -443,6 +534,13 @@ static void hisi_sllc_pmu_remove(struct platform_device *pdev)
&sllc_pmu->node);
}
+static const struct acpi_device_id hisi_sllc_pmu_acpi_match[] = {
+ { "HISI0263", (kernel_ulong_t)&hisi_sllc_v2 },
+ { "HISI0264", (kernel_ulong_t)&hisi_sllc_v3 },
+ {}
+};
+MODULE_DEVICE_TABLE(acpi, hisi_sllc_pmu_acpi_match);
+
static struct platform_driver hisi_sllc_pmu_driver = {
.driver = {
.name = "hisi_sllc_pmu",
generated by cgit (git 2.34.1) at 2025-07-30 20:07:34 +0000