// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2008-2017 Andes Technology Corporation * * Reference ARMv7: Jean Pihet * 2010 (c) MontaVista Software, LLC. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Set at runtime when we know what CPU type we are. */ static struct nds32_pmu *cpu_pmu; static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events); static void nds32_pmu_start(struct nds32_pmu *cpu_pmu); static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu); static struct platform_device_id cpu_pmu_plat_device_ids[] = { {.name = "nds32-pfm"}, {}, }; static int nds32_pmu_map_cache_event(const unsigned int (*cache_map) [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX], u64 config) { unsigned int cache_type, cache_op, cache_result, ret; cache_type = (config >> 0) & 0xff; if (cache_type >= PERF_COUNT_HW_CACHE_MAX) return -EINVAL; cache_op = (config >> 8) & 0xff; if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) return -EINVAL; cache_result = (config >> 16) & 0xff; if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) return -EINVAL; ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; if (ret == CACHE_OP_UNSUPPORTED) return -ENOENT; return ret; } static int nds32_pmu_map_hw_event(const unsigned int (*event_map)[PERF_COUNT_HW_MAX], u64 config) { int mapping; if (config >= PERF_COUNT_HW_MAX) return -ENOENT; mapping = (*event_map)[config]; return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; } static int nds32_pmu_map_raw_event(u32 raw_event_mask, u64 config) { int ev_type = (int)(config & raw_event_mask); int idx = config >> 8; switch (idx) { case 0: ev_type = PFM_OFFSET_MAGIC_0 + ev_type; if (ev_type >= SPAV3_0_SEL_LAST || ev_type <= SPAV3_0_SEL_BASE) return -ENOENT; break; case 1: ev_type = PFM_OFFSET_MAGIC_1 + ev_type; if (ev_type >= SPAV3_1_SEL_LAST || ev_type <= SPAV3_1_SEL_BASE) return -ENOENT; break; case 2: ev_type = PFM_OFFSET_MAGIC_2 + ev_type; if (ev_type >= SPAV3_2_SEL_LAST || ev_type <= SPAV3_2_SEL_BASE) return -ENOENT; break; default: return -ENOENT; } return ev_type; } int nds32_pmu_map_event(struct perf_event *event, const unsigned int (*event_map)[PERF_COUNT_HW_MAX], const unsigned int (*cache_map) [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX], u32 raw_event_mask) { u64 config = event->attr.config; switch (event->attr.type) { case PERF_TYPE_HARDWARE: return nds32_pmu_map_hw_event(event_map, config); case PERF_TYPE_HW_CACHE: return nds32_pmu_map_cache_event(cache_map, config); case PERF_TYPE_RAW: return nds32_pmu_map_raw_event(raw_event_mask, config); } return -ENOENT; } static int nds32_spav3_map_event(struct perf_event *event) { return nds32_pmu_map_event(event, &nds32_pfm_perf_map, &nds32_pfm_perf_cache_map, SOFTWARE_EVENT_MASK); } static inline u32 nds32_pfm_getreset_flags(void) { /* Read overflow status */ u32 val = __nds32__mfsr(NDS32_SR_PFM_CTL); u32 old_val = val; /* Write overflow bit to clear status, and others keep it 0 */ u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]; __nds32__mtsr(val | ov_flag, NDS32_SR_PFM_CTL); return old_val; } static inline int nds32_pfm_has_overflowed(u32 pfm) { u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]; return pfm & ov_flag; } static inline int nds32_pfm_counter_has_overflowed(u32 pfm, int idx) { u32 mask = 0; switch (idx) { case 0: mask = PFM_CTL_OVF[0]; break; case 1: mask = PFM_CTL_OVF[1]; break; case 2: mask = PFM_CTL_OVF[2]; break; default: pr_err("%s index wrong\n", __func__); break; } return pfm & mask; } /* * Set the next IRQ period, based on the hwc->period_left value. * To be called with the event disabled in hw: */ int nds32_pmu_event_set_period(struct perf_event *event) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; s64 left = local64_read(&hwc->period_left); s64 period = hwc->sample_period; int ret = 0; /* The period may have been changed by PERF_EVENT_IOC_PERIOD */ if (unlikely(period != hwc->last_period)) left = period - (hwc->last_period - left); if (unlikely(left <= -period)) { left = period; local64_set(&hwc->period_left, left); hwc->last_period = period; ret = 1; } if (unlikely(left <= 0)) { left += period; local64_set(&hwc->period_left, left); hwc->last_period = period; ret = 1; } if (left > (s64)nds32_pmu->max_period) left = nds32_pmu->max_period; /* * The hw event starts counting from this event offset, * mark it to be able to extract future "deltas": */ local64_set(&hwc->prev_count, (u64)(-left)); nds32_pmu->write_counter(event, (u64)(-left) & nds32_pmu->max_period); perf_event_update_userpage(event); return ret; } static irqreturn_t nds32_pmu_handle_irq(int irq_num, void *dev) { u32 pfm; struct perf_sample_data data; struct nds32_pmu *cpu_pmu = (struct nds32_pmu *)dev; struct pmu_hw_events *cpuc = cpu_pmu->get_hw_events(); struct pt_regs *regs; int idx; /* * Get and reset the IRQ flags */ pfm = nds32_pfm_getreset_flags(); /* * Did an overflow occur? */ if (!nds32_pfm_has_overflowed(pfm)) return IRQ_NONE; /* * Handle the counter(s) overflow(s) */ regs = get_irq_regs(); nds32_pmu_stop(cpu_pmu); for (idx = 0; idx < cpu_pmu->num_events; ++idx) { struct perf_event *event = cpuc->events[idx]; struct hw_perf_event *hwc; /* Ignore if we don't have an event. */ if (!event) continue; /* * We have a single interrupt for all counters. Check that * each counter has overflowed before we process it. */ if (!nds32_pfm_counter_has_overflowed(pfm, idx)) continue; hwc = &event->hw; nds32_pmu_event_update(event); perf_sample_data_init(&data, 0, hwc->last_period); if (!nds32_pmu_event_set_period(event)) continue; if (perf_event_overflow(event, &data, regs)) cpu_pmu->disable(event); } nds32_pmu_start(cpu_pmu); /* * Handle the pending perf events. * * Note: this call *must* be run with interrupts disabled. For * platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ irq_work_run(); return IRQ_HANDLED; } static inline int nds32_pfm_counter_valid(struct nds32_pmu *cpu_pmu, int idx) { return ((idx >= 0) && (idx < cpu_pmu->num_events)); } static inline int nds32_pfm_disable_counter(int idx) { unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); u32 mask = 0; mask = PFM_CTL_EN[idx]; val &= ~mask; val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); return idx; } /* * Add an event filter to a given event. */ static int nds32_pmu_set_event_filter(struct hw_perf_event *event, struct perf_event_attr *attr) { unsigned long config_base = 0; int idx = event->idx; unsigned long no_kernel_tracing = 0; unsigned long no_user_tracing = 0; /* If index is -1, do not do anything */ if (idx == -1) return 0; no_kernel_tracing = PFM_CTL_KS[idx]; no_user_tracing = PFM_CTL_KU[idx]; /* * Default: enable both kernel and user mode tracing. */ if (attr->exclude_user) config_base |= no_user_tracing; if (attr->exclude_kernel) config_base |= no_kernel_tracing; /* * Install the filter into config_base as this is used to * construct the event type. */ event->config_base |= config_base; return 0; } static inline void nds32_pfm_write_evtsel(int idx, u32 evnum) { u32 offset = 0; u32 ori_val = __nds32__mfsr(NDS32_SR_PFM_CTL); u32 ev_mask = 0; u32 no_kernel_mask = 0; u32 no_user_mask = 0; u32 val; offset = PFM_CTL_OFFSEL[idx]; /* Clear previous mode selection, and write new one */ no_kernel_mask = PFM_CTL_KS[idx]; no_user_mask = PFM_CTL_KU[idx]; ori_val &= ~no_kernel_mask; ori_val &= ~no_user_mask; if (evnum & no_kernel_mask) ori_val |= no_kernel_mask; if (evnum & no_user_mask) ori_val |= no_user_mask; /* Clear previous event selection */ ev_mask = PFM_CTL_SEL[idx]; ori_val &= ~ev_mask; evnum &= SOFTWARE_EVENT_MASK; /* undo the linear mapping */ evnum = get_converted_evet_hw_num(evnum); val = ori_val | (evnum << offset); val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); } static inline int nds32_pfm_enable_counter(int idx) { unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); u32 mask = 0; mask = PFM_CTL_EN[idx]; val |= mask; val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); return idx; } static inline int nds32_pfm_enable_intens(int idx) { unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); u32 mask = 0; mask = PFM_CTL_IE[idx]; val |= mask; val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); return idx; } static inline int nds32_pfm_disable_intens(int idx) { unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); u32 mask = 0; mask = PFM_CTL_IE[idx]; val &= ~mask; val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); return idx; } static int event_requires_mode_exclusion(struct perf_event_attr *attr) { /* Other modes NDS32 does not support */ return attr->exclude_user || attr->exclude_kernel; } static void nds32_pmu_enable_event(struct perf_event *event) { unsigned long flags; unsigned int evnum = 0; struct hw_perf_event *hwc = &event->hw; struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); struct pmu_hw_events *events = cpu_pmu->get_hw_events(); int idx = hwc->idx; if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { pr_err("CPU enabling wrong pfm counter IRQ enable\n"); return; } /* * Enable counter and interrupt, and set the counter to count * the event that we're interested in. */ raw_spin_lock_irqsave(&events->pmu_lock, flags); /* * Disable counter */ nds32_pfm_disable_counter(idx); /* * Check whether we need to exclude the counter from certain modes. */ if ((!cpu_pmu->set_event_filter || cpu_pmu->set_event_filter(hwc, &event->attr)) && event_requires_mode_exclusion(&event->attr)) { pr_notice ("NDS32 performance counters do not support mode exclusion\n"); hwc->config_base = 0; } /* Write event */ evnum = hwc->config_base; nds32_pfm_write_evtsel(idx, evnum); /* * Enable interrupt for this counter */ nds32_pfm_enable_intens(idx); /* * Enable counter */ nds32_pfm_enable_counter(idx); raw_spin_unlock_irqrestore(&events->pmu_lock, flags); } static void nds32_pmu_disable_event(struct perf_event *event) { unsigned long flags; struct hw_perf_event *hwc = &event->hw; struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); struct pmu_hw_events *events = cpu_pmu->get_hw_events(); int idx = hwc->idx; if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { pr_err("CPU disabling wrong pfm counter IRQ enable %d\n", idx); return; } /* * Disable counter and interrupt */ raw_spin_lock_irqsave(&events->pmu_lock, flags); /* * Disable counter */ nds32_pfm_disable_counter(idx); /* * Disable interrupt for this counter */ nds32_pfm_disable_intens(idx); raw_spin_unlock_irqrestore(&events->pmu_lock, flags); } static inline u32 nds32_pmu_read_counter(struct perf_event *event) { struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; u32 count = 0; if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { pr_err("CPU reading wrong counter %d\n", idx); } else { switch (idx) { case PFMC0: count = __nds32__mfsr(NDS32_SR_PFMC0); break; case PFMC1: count = __nds32__mfsr(NDS32_SR_PFMC1); break; case PFMC2: count = __nds32__mfsr(NDS32_SR_PFMC2); break; default: pr_err ("%s: CPU has no performance counters %d\n", __func__, idx); } } return count; } static inline void nds32_pmu_write_counter(struct perf_event *event, u32 value) { struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { pr_err("CPU writing wrong counter %d\n", idx); } else { switch (idx) { case PFMC0: __nds32__mtsr_isb(value, NDS32_SR_PFMC0); break; case PFMC1: __nds32__mtsr_isb(value, NDS32_SR_PFMC1); break; case PFMC2: __nds32__mtsr_isb(value, NDS32_SR_PFMC2); break; default: pr_err ("%s: CPU has no performance counters %d\n", __func__, idx); } } } static int nds32_pmu_get_event_idx(struct pmu_hw_events *cpuc, struct perf_event *event) { int idx; struct hw_perf_event *hwc = &event->hw; /* * Current implementation maps cycles, instruction count and cache-miss * to specific counter. * However, multiple of the 3 counters are able to count these events. * * * SOFTWARE_EVENT_MASK mask for getting event num , * This is defined by Jia-Rung, you can change the polocies. * However, do not exceed 8 bits. This is hardware specific. * The last number is SPAv3_2_SEL_LAST. */ unsigned long evtype = hwc->config_base & SOFTWARE_EVENT_MASK; idx = get_converted_event_idx(evtype); /* * Try to get the counter for correpsonding event */ if (evtype == SPAV3_0_SEL_TOTAL_CYCLES) { if (!test_and_set_bit(idx, cpuc->used_mask)) return idx; if (!test_and_set_bit(NDS32_IDX_COUNTER0, cpuc->used_mask)) return NDS32_IDX_COUNTER0; if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask)) return NDS32_IDX_COUNTER1; } else if (evtype == SPAV3_1_SEL_COMPLETED_INSTRUCTION) { if (!test_and_set_bit(idx, cpuc->used_mask)) return idx; else if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask)) return NDS32_IDX_COUNTER1; else if (!test_and_set_bit (NDS32_IDX_CYCLE_COUNTER, cpuc->used_mask)) return NDS32_IDX_CYCLE_COUNTER; } else { if (!test_and_set_bit(idx, cpuc->used_mask)) return idx; } return -EAGAIN; } static void nds32_pmu_start(struct nds32_pmu *cpu_pmu) { unsigned long flags; unsigned int val; struct pmu_hw_events *events = cpu_pmu->get_hw_events(); raw_spin_lock_irqsave(&events->pmu_lock, flags); /* Enable all counters , NDS PFM has 3 counters */ val = __nds32__mfsr(NDS32_SR_PFM_CTL); val |= (PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]); val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); raw_spin_unlock_irqrestore(&events->pmu_lock, flags); } static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu) { unsigned long flags; unsigned int val; struct pmu_hw_events *events = cpu_pmu->get_hw_events(); raw_spin_lock_irqsave(&events->pmu_lock, flags); /* Disable all counters , NDS PFM has 3 counters */ val = __nds32__mfsr(NDS32_SR_PFM_CTL); val &= ~(PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]); val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); raw_spin_unlock_irqrestore(&events->pmu_lock, flags); } static void nds32_pmu_reset(void *info) { u32 val = 0; val |= (PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); __nds32__mtsr(val, NDS32_SR_PFM_CTL); __nds32__mtsr(0, NDS32_SR_PFM_CTL); __nds32__mtsr(0, NDS32_SR_PFMC0); __nds32__mtsr(0, NDS32_SR_PFMC1); __nds32__mtsr(0, NDS32_SR_PFMC2); } static void nds32_pmu_init(struct nds32_pmu *cpu_pmu) { cpu_pmu->handle_irq = nds32_pmu_handle_irq; cpu_pmu->enable = nds32_pmu_enable_event; cpu_pmu->disable = nds32_pmu_disable_event; cpu_pmu->read_counter = nds32_pmu_read_counter; cpu_pmu->write_counter = nds32_pmu_write_counter; cpu_pmu->get_event_idx = nds32_pmu_get_event_idx; cpu_pmu->start = nds32_pmu_start; cpu_pmu->stop = nds32_pmu_stop; cpu_pmu->reset = nds32_pmu_reset; cpu_pmu->max_period = 0xFFFFFFFF; /* Maximum counts */ }; static u32 nds32_read_num_pfm_events(void) { /* NDS32 SPAv3 PMU support 3 counter */ return 3; } static int device_pmu_init(struct nds32_pmu *cpu_pmu) { nds32_pmu_init(cpu_pmu); /* * This name should be devive-specific name, whatever you like :) * I think "PMU" will be a good generic name. */ cpu_pmu->name = "nds32v3-pmu"; cpu_pmu->map_event = nds32_spav3_map_event; cpu_pmu->num_events = nds32_read_num_pfm_events(); cpu_pmu->set_event_filter = nds32_pmu_set_event_filter; return 0; } /* * CPU PMU identification and probing. */ static int probe_current_pmu(struct nds32_pmu *pmu) { int ret; get_cpu(); ret = -ENODEV; /* * If ther are various CPU types with its own PMU, initialize with * * the corresponding one */ device_pmu_init(pmu); put_cpu(); return ret; } static void nds32_pmu_enable(struct pmu *pmu) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu); struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); int enabled = bitmap_weight(hw_events->used_mask, nds32_pmu->num_events); if (enabled) nds32_pmu->start(nds32_pmu); } static void nds32_pmu_disable(struct pmu *pmu) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu); nds32_pmu->stop(nds32_pmu); } static void nds32_pmu_release_hardware(struct nds32_pmu *nds32_pmu) { nds32_pmu->free_irq(nds32_pmu); pm_runtime_put_sync(&nds32_pmu->plat_device->dev); } static irqreturn_t nds32_pmu_dispatch_irq(int irq, void *dev) { struct nds32_pmu *nds32_pmu = (struct nds32_pmu *)dev; int ret; u64 start_clock, finish_clock; start_clock = local_clock(); ret = nds32_pmu->handle_irq(irq, dev); finish_clock = local_clock(); perf_sample_event_took(finish_clock - start_clock); return ret; } static int nds32_pmu_reserve_hardware(struct nds32_pmu *nds32_pmu) { int err; struct platform_device *pmu_device = nds32_pmu->plat_device; if (!pmu_device) return -ENODEV; pm_runtime_get_sync(&pmu_device->dev); err = nds32_pmu->request_irq(nds32_pmu, nds32_pmu_dispatch_irq); if (err) { nds32_pmu_release_hardware(nds32_pmu); return err; } return 0; } static int validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, struct perf_event *event) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); if (is_software_event(event)) return 1; if (event->pmu != pmu) return 0; if (event->state < PERF_EVENT_STATE_OFF) return 1; if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) return 1; return nds32_pmu->get_event_idx(hw_events, event) >= 0; } static int validate_group(struct perf_event *event) { struct perf_event *sibling, *leader = event->group_leader; struct pmu_hw_events fake_pmu; DECLARE_BITMAP(fake_used_mask, MAX_COUNTERS); /* * Initialize the fake PMU. We only need to populate the * used_mask for the purposes of validation. */ memset(fake_used_mask, 0, sizeof(fake_used_mask)); if (!validate_event(event->pmu, &fake_pmu, leader)) return -EINVAL; for_each_sibling_event(sibling, leader) { if (!validate_event(event->pmu, &fake_pmu, sibling)) return -EINVAL; } if (!validate_event(event->pmu, &fake_pmu, event)) return -EINVAL; return 0; } static int __hw_perf_event_init(struct perf_event *event) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; int mapping; mapping = nds32_pmu->map_event(event); if (mapping < 0) { pr_debug("event %x:%llx not supported\n", event->attr.type, event->attr.config); return mapping; } /* * We don't assign an index until we actually place the event onto * hardware. Use -1 to signify that we haven't decided where to put it * yet. For SMP systems, each core has it's own PMU so we can't do any * clever allocation or constraints checking at this point. */ hwc->idx = -1; hwc->config_base = 0; hwc->config = 0; hwc->event_base = 0; /* * Check whether we need to exclude the counter from certain modes. */ if ((!nds32_pmu->set_event_filter || nds32_pmu->set_event_filter(hwc, &event->attr)) && event_requires_mode_exclusion(&event->attr)) { pr_debug ("NDS performance counters do not support mode exclusion\n"); return -EOPNOTSUPP; } /* * Store the event encoding into the config_base field. */ hwc->config_base |= (unsigned long)mapping; if (!hwc->sample_period) { /* * For non-sampling runs, limit the sample_period to half * of the counter width. That way, the new counter value * is far less likely to overtake the previous one unless * you have some serious IRQ latency issues. */ hwc->sample_period = nds32_pmu->max_period >> 1; hwc->last_period = hwc->sample_period; local64_set(&hwc->period_left, hwc->sample_period); } if (event->group_leader != event) { if (validate_group(event) != 0) return -EINVAL; } return 0; } static int nds32_pmu_event_init(struct perf_event *event) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); int err = 0; atomic_t *active_events = &nds32_pmu->active_events; /* does not support taken branch sampling */ if (has_branch_stack(event)) return -EOPNOTSUPP; if (nds32_pmu->map_event(event) == -ENOENT) return -ENOENT; if (!atomic_inc_not_zero(active_events)) { if (atomic_read(active_events) == 0) { /* Register irq handler */ err = nds32_pmu_reserve_hardware(nds32_pmu); } if (!err) atomic_inc(active_events); } if (err) return err; err = __hw_perf_event_init(event); return err; } static void nds32_start(struct perf_event *event, int flags) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; /* * NDS pmu always has to reprogram the period, so ignore * PERF_EF_RELOAD, see the comment below. */ if (flags & PERF_EF_RELOAD) WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); hwc->state = 0; /* Set the period for the event. */ nds32_pmu_event_set_period(event); nds32_pmu->enable(event); } static int nds32_pmu_add(struct perf_event *event, int flags) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); struct hw_perf_event *hwc = &event->hw; int idx; int err = 0; perf_pmu_disable(event->pmu); /* If we don't have a space for the counter then finish early. */ idx = nds32_pmu->get_event_idx(hw_events, event); if (idx < 0) { err = idx; goto out; } /* * If there is an event in the counter we are going to use then make * sure it is disabled. */ event->hw.idx = idx; nds32_pmu->disable(event); hw_events->events[idx] = event; hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; if (flags & PERF_EF_START) nds32_start(event, PERF_EF_RELOAD); /* Propagate our changes to the userspace mapping. */ perf_event_update_userpage(event); out: perf_pmu_enable(event->pmu); return err; } u64 nds32_pmu_event_update(struct perf_event *event) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; u64 delta, prev_raw_count, new_raw_count; again: prev_raw_count = local64_read(&hwc->prev_count); new_raw_count = nds32_pmu->read_counter(event); if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) { goto again; } /* * Whether overflow or not, "unsigned substraction" * will always get their delta */ delta = (new_raw_count - prev_raw_count) & nds32_pmu->max_period; local64_add(delta, &event->count); local64_sub(delta, &hwc->period_left); return new_raw_count; } static void nds32_stop(struct perf_event *event, int flags) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; /* * NDS pmu always has to update the counter, so ignore * PERF_EF_UPDATE, see comments in nds32_start(). */ if (!(hwc->state & PERF_HES_STOPPED)) { nds32_pmu->disable(event); nds32_pmu_event_update(event); hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } } static void nds32_pmu_del(struct perf_event *event, int flags) { struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; nds32_stop(event, PERF_EF_UPDATE); hw_events->events[idx] = NULL; clear_bit(idx, hw_events->used_mask); perf_event_update_userpage(event); } static void nds32_pmu_read(struct perf_event *event) { nds32_pmu_event_update(event); } /* Please refer to SPAv3 for more hardware specific details */ PMU_FORMAT_ATTR(event, "config:0-63"); static struct attribute *nds32_arch_formats_attr[] = { &format_attr_event.attr, NULL, }; static struct attribute_group nds32_pmu_format_group = { .name = "format", .attrs = nds32_arch_formats_attr, }; static ssize_t nds32_pmu_cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) { return 0; } static DEVICE_ATTR(cpus, 0444, nds32_pmu_cpumask_show, NULL); static struct attribute *nds32_pmu_common_attrs[] = { &dev_attr_cpus.attr, NULL, }; static struct attribute_group nds32_pmu_common_group = { .attrs = nds32_pmu_common_attrs, }; static const struct attribute_group *nds32_pmu_attr_groups[] = { &nds32_pmu_format_group, &nds32_pmu_common_group, NULL, }; static void nds32_init(struct nds32_pmu *nds32_pmu) { atomic_set(&nds32_pmu->active_events, 0); nds32_pmu->pmu = (struct pmu) { .pmu_enable = nds32_pmu_enable, .pmu_disable = nds32_pmu_disable, .attr_groups = nds32_pmu_attr_groups, .event_init = nds32_pmu_event_init, .add = nds32_pmu_add, .del = nds32_pmu_del, .start = nds32_start, .stop = nds32_stop, .read = nds32_pmu_read, }; } int nds32_pmu_register(struct nds32_pmu *nds32_pmu, int type) { nds32_init(nds32_pmu); pm_runtime_enable(&nds32_pmu->plat_device->dev); pr_info("enabled with %s PMU driver, %d counters available\n", nds32_pmu->name, nds32_pmu->num_events); return perf_pmu_register(&nds32_pmu->pmu, nds32_pmu->name, type); } static struct pmu_hw_events *cpu_pmu_get_cpu_events(void) { return this_cpu_ptr(&cpu_hw_events); } static int cpu_pmu_request_irq(struct nds32_pmu *cpu_pmu, irq_handler_t handler) { int err, irq, irqs; struct platform_device *pmu_device = cpu_pmu->plat_device; if (!pmu_device) return -ENODEV; irqs = min(pmu_device->num_resources, num_possible_cpus()); if (irqs < 1) { pr_err("no irqs for PMUs defined\n"); return -ENODEV; } irq = platform_get_irq(pmu_device, 0); err = request_irq(irq, handler, IRQF_NOBALANCING, "nds32-pfm", cpu_pmu); if (err) { pr_err("unable to request IRQ%d for NDS PMU counters\n", irq); return err; } return 0; } static void cpu_pmu_free_irq(struct nds32_pmu *cpu_pmu) { int irq; struct platform_device *pmu_device = cpu_pmu->plat_device; irq = platform_get_irq(pmu_device, 0); if (irq >= 0) free_irq(irq, cpu_pmu); } static void cpu_pmu_init(struct nds32_pmu *cpu_pmu) { int cpu; struct pmu_hw_events *events = &per_cpu(cpu_hw_events, cpu); raw_spin_lock_init(&events->pmu_lock); cpu_pmu->get_hw_events = cpu_pmu_get_cpu_events; cpu_pmu->request_irq = cpu_pmu_request_irq; cpu_pmu->free_irq = cpu_pmu_free_irq; /* Ensure the PMU has sane values out of reset. */ if (cpu_pmu->reset) on_each_cpu(cpu_pmu->reset, cpu_pmu, 1); } const static struct of_device_id cpu_pmu_of_device_ids[] = { {.compatible = "andestech,nds32v3-pmu", .data = device_pmu_init}, {}, }; static int cpu_pmu_device_probe(struct platform_device *pdev) { const struct of_device_id *of_id; int (*init_fn)(struct nds32_pmu *nds32_pmu); struct device_node *node = pdev->dev.of_node; struct nds32_pmu *pmu; int ret = -ENODEV; if (cpu_pmu) { pr_notice("[perf] attempt to register multiple PMU devices!\n"); return -ENOSPC; } pmu = kzalloc(sizeof(*pmu), GFP_KERNEL); if (!pmu) return -ENOMEM; of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node); if (node && of_id) { init_fn = of_id->data; ret = init_fn(pmu); } else { ret = probe_current_pmu(pmu); } if (ret) { pr_notice("[perf] failed to probe PMU!\n"); goto out_free; } cpu_pmu = pmu; cpu_pmu->plat_device = pdev; cpu_pmu_init(cpu_pmu); ret = nds32_pmu_register(cpu_pmu, PERF_TYPE_RAW); if (!ret) return 0; out_free: pr_notice("[perf] failed to register PMU devices!\n"); kfree(pmu); return ret; } static struct platform_driver cpu_pmu_driver = { .driver = { .name = "nds32-pfm", .of_match_table = cpu_pmu_of_device_ids, }, .probe = cpu_pmu_device_probe, .id_table = cpu_pmu_plat_device_ids, }; static int __init register_pmu_driver(void) { int err = 0; err = platform_driver_register(&cpu_pmu_driver); if (err) pr_notice("[perf] PMU initialization failed\n"); else pr_notice("[perf] PMU initialization done\n"); return err; } device_initcall(register_pmu_driver); /* * References: arch/nds32/kernel/traps.c:__dump() * You will need to know the NDS ABI first. */ static int unwind_frame_kernel(struct stackframe *frame) { int graph = 0; #ifdef CONFIG_FRAME_POINTER /* 0x3 means misalignment */ if (!kstack_end((void *)frame->fp) && !((unsigned long)frame->fp & 0x3) && ((unsigned long)frame->fp >= TASK_SIZE)) { /* * The array index is based on the ABI, the below graph * illustrate the reasons. * Function call procedure: "smw" and "lmw" will always * update SP and FP for you automatically. * * Stack Relative Address * | | 0 * ---- * |LP| <-- SP(before smw) <-- FP(after smw) -1 * ---- * |FP| -2 * ---- * | | <-- SP(after smw) -3 */ frame->lp = ((unsigned long *)frame->fp)[-1]; frame->fp = ((unsigned long *)frame->fp)[FP_OFFSET]; /* make sure CONFIG_FUNCTION_GRAPH_TRACER is turned on */ if (__kernel_text_address(frame->lp)) frame->lp = ftrace_graph_ret_addr (NULL, &graph, frame->lp, NULL); return 0; } else { return -EPERM; } #else /* * You can refer to arch/nds32/kernel/traps.c:__dump() * Treat "sp" as "fp", but the "sp" is one frame ahead of "fp". * And, the "sp" is not always correct. * * Stack Relative Address * | | 0 * ---- * |LP| <-- SP(before smw) -1 * ---- * | | <-- SP(after smw) -2 * ---- */ if (!kstack_end((void *)frame->sp)) { frame->lp = ((unsigned long *)frame->sp)[1]; /* TODO: How to deal with the value in first * "sp" is not correct? */ if (__kernel_text_address(frame->lp)) frame->lp = ftrace_graph_ret_addr (tsk, &graph, frame->lp, NULL); frame->sp = ((unsigned long *)frame->sp) + 1; return 0; } else { return -EPERM; } #endif } static void notrace walk_stackframe(struct stackframe *frame, int (*fn_record)(struct stackframe *, void *), void *data) { while (1) { int ret; if (fn_record(frame, data)) break; ret = unwind_frame_kernel(frame); if (ret < 0) break; } } /* * Gets called by walk_stackframe() for every stackframe. This will be called * whist unwinding the stackframe and is like a subroutine return so we use * the PC. */ static int callchain_trace(struct stackframe *fr, void *data) { struct perf_callchain_entry_ctx *entry = data; perf_callchain_store(entry, fr->lp); return 0; } /* * Get the return address for a single stackframe and return a pointer to the * next frame tail. */ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp) { struct frame_tail buftail; unsigned long lp = 0; unsigned long *user_frame_tail = (unsigned long *)(fp - (unsigned long)sizeof(buftail)); /* Check accessibility of one struct frame_tail beyond */ if (!access_ok(user_frame_tail, sizeof(buftail))) return 0; if (__copy_from_user_inatomic (&buftail, user_frame_tail, sizeof(buftail))) return 0; /* * Refer to unwind_frame_kernel() for more illurstration */ lp = buftail.stack_lp; /* ((unsigned long *)fp)[-1] */ fp = buftail.stack_fp; /* ((unsigned long *)fp)[FP_OFFSET] */ perf_callchain_store(entry, lp); return fp; } static unsigned long user_backtrace_opt_size(struct perf_callchain_entry_ctx *entry, unsigned long fp) { struct frame_tail_opt_size buftail; unsigned long lp = 0; unsigned long *user_frame_tail = (unsigned long *)(fp - (unsigned long)sizeof(buftail)); /* Check accessibility of one struct frame_tail beyond */ if (!access_ok(user_frame_tail, sizeof(buftail))) return 0; if (__copy_from_user_inatomic (&buftail, user_frame_tail, sizeof(buftail))) return 0; /* * Refer to unwind_frame_kernel() for more illurstration */ lp = buftail.stack_lp; /* ((unsigned long *)fp)[-1] */ fp = buftail.stack_fp; /* ((unsigned long *)fp)[FP_OFFSET] */ perf_callchain_store(entry, lp); return fp; } /* * This will be called when the target is in user mode * This function will only be called when we use * "PERF_SAMPLE_CALLCHAIN" in * kernel/events/core.c:perf_prepare_sample() * * How to trigger perf_callchain_[user/kernel] : * $ perf record -e cpu-clock --call-graph fp ./program * $ perf report --call-graph */ unsigned long leaf_fp; void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { unsigned long fp = 0; unsigned long gp = 0; unsigned long lp = 0; unsigned long sp = 0; unsigned long *user_frame_tail; leaf_fp = 0; if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } perf_callchain_store(entry, regs->ipc); fp = regs->fp; gp = regs->gp; lp = regs->lp; sp = regs->sp; if (entry->nr < PERF_MAX_STACK_DEPTH && (unsigned long)fp && !((unsigned long)fp & 0x7) && fp > sp) { user_frame_tail = (unsigned long *)(fp - (unsigned long)sizeof(fp)); if (!access_ok(user_frame_tail, sizeof(fp))) return; if (__copy_from_user_inatomic (&leaf_fp, user_frame_tail, sizeof(fp))) return; if (leaf_fp == lp) { /* * Maybe this is non leaf function * with optimize for size, * or maybe this is the function * with optimize for size */ struct frame_tail buftail; user_frame_tail = (unsigned long *)(fp - (unsigned long)sizeof(buftail)); if (!access_ok(user_frame_tail, sizeof(buftail))) return; if (__copy_from_user_inatomic (&buftail, user_frame_tail, sizeof(buftail))) return; if (buftail.stack_fp == gp) { /* non leaf function with optimize * for size condition */ struct frame_tail_opt_size buftail_opt_size; user_frame_tail = (unsigned long *)(fp - (unsigned long) sizeof(buftail_opt_size)); if (!access_ok(user_frame_tail, sizeof(buftail_opt_size))) return; if (__copy_from_user_inatomic (&buftail_opt_size, user_frame_tail, sizeof(buftail_opt_size))) return; perf_callchain_store(entry, lp); fp = buftail_opt_size.stack_fp; while ((entry->nr < PERF_MAX_STACK_DEPTH) && (unsigned long)fp && !((unsigned long)fp & 0x7) && fp > sp) { sp = fp; fp = user_backtrace_opt_size(entry, fp); } } else { /* this is the function * without optimize for size */ fp = buftail.stack_fp; perf_callchain_store(entry, lp); while ((entry->nr < PERF_MAX_STACK_DEPTH) && (unsigned long)fp && !((unsigned long)fp & 0x7) && fp > sp) { sp = fp; fp = user_backtrace(entry, fp); } } } else { /* this is leaf function */ fp = leaf_fp; perf_callchain_store(entry, lp); /* previous function callcahin */ while ((entry->nr < PERF_MAX_STACK_DEPTH) && (unsigned long)fp && !((unsigned long)fp & 0x7) && fp > sp) { sp = fp; fp = user_backtrace(entry, fp); } } return; } } /* This will be called when the target is in kernel mode */ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { struct stackframe fr; if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } fr.fp = regs->fp; fr.lp = regs->lp; fr.sp = regs->sp; walk_stackframe(&fr, callchain_trace, entry); } unsigned long perf_instruction_pointer(struct pt_regs *regs) { /* However, NDS32 does not support virtualization */ if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) return perf_guest_cbs->get_guest_ip(); return instruction_pointer(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { int misc = 0; /* However, NDS32 does not support virtualization */ if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { if (perf_guest_cbs->is_user_mode()) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; } else { if (user_mode(regs)) misc |= PERF_RECORD_MISC_USER; else misc |= PERF_RECORD_MISC_KERNEL; } return misc; }