|
@@ -17,7 +17,6 @@
|
|
|
#include <linux/export.h>
|
|
|
#include <linux/kernel.h>
|
|
|
#include <linux/perf/arm_pmu.h>
|
|
|
-#include <linux/platform_device.h>
|
|
|
#include <linux/slab.h>
|
|
|
#include <linux/sched/clock.h>
|
|
|
#include <linux/spinlock.h>
|
|
@@ -26,6 +25,9 @@
|
|
|
|
|
|
#include <asm/irq_regs.h>
|
|
|
|
|
|
+static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu);
|
|
|
+static DEFINE_PER_CPU(int, cpu_irq);
|
|
|
+
|
|
|
static int
|
|
|
armpmu_map_cache_event(const unsigned (*cache_map)
|
|
|
[PERF_COUNT_HW_CACHE_MAX]
|
|
@@ -320,17 +322,9 @@ validate_group(struct perf_event *event)
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-static struct arm_pmu_platdata *armpmu_get_platdata(struct arm_pmu *armpmu)
|
|
|
-{
|
|
|
- struct platform_device *pdev = armpmu->plat_device;
|
|
|
-
|
|
|
- return pdev ? dev_get_platdata(&pdev->dev) : NULL;
|
|
|
-}
|
|
|
-
|
|
|
static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
|
|
|
{
|
|
|
struct arm_pmu *armpmu;
|
|
|
- struct arm_pmu_platdata *plat;
|
|
|
int ret;
|
|
|
u64 start_clock, finish_clock;
|
|
|
|
|
@@ -341,14 +335,11 @@ static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
|
|
|
* dereference.
|
|
|
*/
|
|
|
armpmu = *(void **)dev;
|
|
|
-
|
|
|
- plat = armpmu_get_platdata(armpmu);
|
|
|
+ if (WARN_ON_ONCE(!armpmu))
|
|
|
+ return IRQ_NONE;
|
|
|
|
|
|
start_clock = sched_clock();
|
|
|
- if (plat && plat->handle_irq)
|
|
|
- ret = plat->handle_irq(irq, armpmu, armpmu->handle_irq);
|
|
|
- else
|
|
|
- ret = armpmu->handle_irq(irq, armpmu);
|
|
|
+ ret = armpmu->handle_irq(irq, armpmu);
|
|
|
finish_clock = sched_clock();
|
|
|
|
|
|
perf_sample_event_took(finish_clock - start_clock);
|
|
@@ -531,54 +522,41 @@ int perf_num_counters(void)
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(perf_num_counters);
|
|
|
|
|
|
-void armpmu_free_irq(struct arm_pmu *armpmu, int cpu)
|
|
|
+static int armpmu_count_irq_users(const int irq)
|
|
|
{
|
|
|
- struct pmu_hw_events __percpu *hw_events = armpmu->hw_events;
|
|
|
- int irq = per_cpu(hw_events->irq, cpu);
|
|
|
+ int cpu, count = 0;
|
|
|
|
|
|
- if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))
|
|
|
- return;
|
|
|
-
|
|
|
- if (irq_is_percpu_devid(irq)) {
|
|
|
- free_percpu_irq(irq, &hw_events->percpu_pmu);
|
|
|
- cpumask_clear(&armpmu->active_irqs);
|
|
|
- return;
|
|
|
+ for_each_possible_cpu(cpu) {
|
|
|
+ if (per_cpu(cpu_irq, cpu) == irq)
|
|
|
+ count++;
|
|
|
}
|
|
|
|
|
|
- free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));
|
|
|
+ return count;
|
|
|
}
|
|
|
|
|
|
-void armpmu_free_irqs(struct arm_pmu *armpmu)
|
|
|
+void armpmu_free_irq(int irq, int cpu)
|
|
|
{
|
|
|
- int cpu;
|
|
|
+ if (per_cpu(cpu_irq, cpu) == 0)
|
|
|
+ return;
|
|
|
+ if (WARN_ON(irq != per_cpu(cpu_irq, cpu)))
|
|
|
+ return;
|
|
|
+
|
|
|
+ if (!irq_is_percpu_devid(irq))
|
|
|
+ free_irq(irq, per_cpu_ptr(&cpu_armpmu, cpu));
|
|
|
+ else if (armpmu_count_irq_users(irq) == 1)
|
|
|
+ free_percpu_irq(irq, &cpu_armpmu);
|
|
|
|
|
|
- for_each_cpu(cpu, &armpmu->supported_cpus)
|
|
|
- armpmu_free_irq(armpmu, cpu);
|
|
|
+ per_cpu(cpu_irq, cpu) = 0;
|
|
|
}
|
|
|
|
|
|
-int armpmu_request_irq(struct arm_pmu *armpmu, int cpu)
|
|
|
+int armpmu_request_irq(int irq, int cpu)
|
|
|
{
|
|
|
int err = 0;
|
|
|
- struct pmu_hw_events __percpu *hw_events = armpmu->hw_events;
|
|
|
const irq_handler_t handler = armpmu_dispatch_irq;
|
|
|
- int irq = per_cpu(hw_events->irq, cpu);
|
|
|
if (!irq)
|
|
|
return 0;
|
|
|
|
|
|
- if (irq_is_percpu_devid(irq) && cpumask_empty(&armpmu->active_irqs)) {
|
|
|
- err = request_percpu_irq(irq, handler, "arm-pmu",
|
|
|
- &hw_events->percpu_pmu);
|
|
|
- } else if (irq_is_percpu_devid(irq)) {
|
|
|
- int other_cpu = cpumask_first(&armpmu->active_irqs);
|
|
|
- int other_irq = per_cpu(hw_events->irq, other_cpu);
|
|
|
-
|
|
|
- if (irq != other_irq) {
|
|
|
- pr_warn("mismatched PPIs detected.\n");
|
|
|
- err = -EINVAL;
|
|
|
- goto err_out;
|
|
|
- }
|
|
|
- } else {
|
|
|
- struct arm_pmu_platdata *platdata = armpmu_get_platdata(armpmu);
|
|
|
+ if (!irq_is_percpu_devid(irq)) {
|
|
|
unsigned long irq_flags;
|
|
|
|
|
|
err = irq_force_affinity(irq, cpumask_of(cpu));
|
|
@@ -589,22 +567,22 @@ int armpmu_request_irq(struct arm_pmu *armpmu, int cpu)
|
|
|
goto err_out;
|
|
|
}
|
|
|
|
|
|
- if (platdata && platdata->irq_flags) {
|
|
|
- irq_flags = platdata->irq_flags;
|
|
|
- } else {
|
|
|
- irq_flags = IRQF_PERCPU |
|
|
|
- IRQF_NOBALANCING |
|
|
|
- IRQF_NO_THREAD;
|
|
|
- }
|
|
|
+ irq_flags = IRQF_PERCPU |
|
|
|
+ IRQF_NOBALANCING |
|
|
|
+ IRQF_NO_THREAD;
|
|
|
|
|
|
+ irq_set_status_flags(irq, IRQ_NOAUTOEN);
|
|
|
err = request_irq(irq, handler, irq_flags, "arm-pmu",
|
|
|
- per_cpu_ptr(&hw_events->percpu_pmu, cpu));
|
|
|
+ per_cpu_ptr(&cpu_armpmu, cpu));
|
|
|
+ } else if (armpmu_count_irq_users(irq) == 0) {
|
|
|
+ err = request_percpu_irq(irq, handler, "arm-pmu",
|
|
|
+ &cpu_armpmu);
|
|
|
}
|
|
|
|
|
|
if (err)
|
|
|
goto err_out;
|
|
|
|
|
|
- cpumask_set_cpu(cpu, &armpmu->active_irqs);
|
|
|
+ per_cpu(cpu_irq, cpu) = irq;
|
|
|
return 0;
|
|
|
|
|
|
err_out:
|
|
@@ -612,19 +590,6 @@ err_out:
|
|
|
return err;
|
|
|
}
|
|
|
|
|
|
-int armpmu_request_irqs(struct arm_pmu *armpmu)
|
|
|
-{
|
|
|
- int cpu, err;
|
|
|
-
|
|
|
- for_each_cpu(cpu, &armpmu->supported_cpus) {
|
|
|
- err = armpmu_request_irq(armpmu, cpu);
|
|
|
- if (err)
|
|
|
- break;
|
|
|
- }
|
|
|
-
|
|
|
- return err;
|
|
|
-}
|
|
|
-
|
|
|
static int armpmu_get_cpu_irq(struct arm_pmu *pmu, int cpu)
|
|
|
{
|
|
|
struct pmu_hw_events __percpu *hw_events = pmu->hw_events;
|
|
@@ -647,12 +612,14 @@ static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node)
|
|
|
if (pmu->reset)
|
|
|
pmu->reset(pmu);
|
|
|
|
|
|
+ per_cpu(cpu_armpmu, cpu) = pmu;
|
|
|
+
|
|
|
irq = armpmu_get_cpu_irq(pmu, cpu);
|
|
|
if (irq) {
|
|
|
- if (irq_is_percpu_devid(irq)) {
|
|
|
+ if (irq_is_percpu_devid(irq))
|
|
|
enable_percpu_irq(irq, IRQ_TYPE_NONE);
|
|
|
- return 0;
|
|
|
- }
|
|
|
+ else
|
|
|
+ enable_irq(irq);
|
|
|
}
|
|
|
|
|
|
return 0;
|
|
@@ -667,8 +634,14 @@ static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node)
|
|
|
return 0;
|
|
|
|
|
|
irq = armpmu_get_cpu_irq(pmu, cpu);
|
|
|
- if (irq && irq_is_percpu_devid(irq))
|
|
|
- disable_percpu_irq(irq);
|
|
|
+ if (irq) {
|
|
|
+ if (irq_is_percpu_devid(irq))
|
|
|
+ disable_percpu_irq(irq);
|
|
|
+ else
|
|
|
+ disable_irq(irq);
|
|
|
+ }
|
|
|
+
|
|
|
+ per_cpu(cpu_armpmu, cpu) = NULL;
|
|
|
|
|
|
return 0;
|
|
|
}
|
|
@@ -800,18 +773,18 @@ static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)
|
|
|
&cpu_pmu->node);
|
|
|
}
|
|
|
|
|
|
-struct arm_pmu *armpmu_alloc(void)
|
|
|
+static struct arm_pmu *__armpmu_alloc(gfp_t flags)
|
|
|
{
|
|
|
struct arm_pmu *pmu;
|
|
|
int cpu;
|
|
|
|
|
|
- pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
|
|
|
+ pmu = kzalloc(sizeof(*pmu), flags);
|
|
|
if (!pmu) {
|
|
|
pr_info("failed to allocate PMU device!\n");
|
|
|
goto out;
|
|
|
}
|
|
|
|
|
|
- pmu->hw_events = alloc_percpu(struct pmu_hw_events);
|
|
|
+ pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, flags);
|
|
|
if (!pmu->hw_events) {
|
|
|
pr_info("failed to allocate per-cpu PMU data.\n");
|
|
|
goto out_free_pmu;
|
|
@@ -857,6 +830,17 @@ out:
|
|
|
return NULL;
|
|
|
}
|
|
|
|
|
|
+struct arm_pmu *armpmu_alloc(void)
|
|
|
+{
|
|
|
+ return __armpmu_alloc(GFP_KERNEL);
|
|
|
+}
|
|
|
+
|
|
|
+struct arm_pmu *armpmu_alloc_atomic(void)
|
|
|
+{
|
|
|
+ return __armpmu_alloc(GFP_ATOMIC);
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
void armpmu_free(struct arm_pmu *pmu)
|
|
|
{
|
|
|
free_percpu(pmu->hw_events);
|