Skip to content

Instantly share code, notes, and snippets.

@cyring
Last active May 9, 2024 18:57
Show Gist options
  • Save cyring/14556fc52e5bea5a564bdda5a0852f09 to your computer and use it in GitHub Desktop.
Save cyring/14556fc52e5bea5a564bdda5a0852f09 to your computer and use it in GitHub Desktop.
ARM PMU
Orange Pi 5+ (DTS)
@cyring
Copy link
Author

cyring commented Dec 26, 2023

volatile unsigned long long 
USER, SELR, XEVT, EVT0, ESET, ECLR, PMCR, FLTR, OVF, INT;

__asm__ __volatile__
(
        "mrs    %[USER] ,       pmuserenr_el0"  "\n\t"
        "mrs    %[SELR] ,       pmselr_el0"     "\n\t"
        "mrs    %[XEVT] ,       pmxevtyper_el0" "\n\t"
        "mrs    %[EVT0] ,       pmevtyper0_el0" "\n\t"
        "mrs    %[ESET] ,       pmcntenset_el0" "\n\t"
        "mrs    %[ECLR] ,       pmcntenclr_el0" "\n\t"
        "mrs    %[PMCR] ,       pmcr_el0"       "\n\t"
        "mrs    %[FLTR] ,       pmccfiltr_el0"  "\n\t"
        "mrs    %[OVF]  ,       pmovsset_el0"   "\n\t"
        "mrs    %[INT]  ,       pmintenset_el1" "\n\t"
        "isb"
        : [USER] "=r" (USER),
          [SELR] "=r" (SELR),
          [XEVT] "=r" (XEVT),
          [EVT0] "=r" (EVT0),
          [ESET] "=r" (ESET),
          [ECLR] "=r" (ECLR),
          [PMCR] "=r" (PMCR),
          [FLTR] "=r" (FLTR),
          [OVF]  "=r" (OVF),
          [INT]  "=r" (INT)
        :
        : "memory"
);
printk( "CPU[%03d]\n" \
        "pmuserenr [%016llx]\n" "pmselr    [%016llx]\n" \
        "pmxevtyper[%016llx]\n" "pmevtyper0[%016llx]\n" \
        "pmcntenset[%016llx]\n" "pmcntenclr[%016llx]\n" \
        "pmcr      [%016llx]\n" "pmccfiltr [%016llx]\n" \
        "pmovsset  [%016llx]\n" "pmintenset[%016llx]\n",
        Core->Bind,
        USER, SELR, XEVT, EVT0, ESET, ECLR, PMCR, FLTR, OVF, INT);
perf record -e instructions
CPU[001]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [0000000041453041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000000]
pmintenset[0000000000000001]
CPU[000]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [0000000041453041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000000]
pmintenset[0000000000000001]
CPU[002]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [0000000041453041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000000]
pmintenset[0000000000000001]
CPU[003]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [0000000041453041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000000]
pmintenset[0000000000000001]
CPU[004]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [00000000410b3041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000000]
pmintenset[0000000000000001]
CPU[005]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [00000000410b3041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000001]
pmintenset[0000000000000001]
CPU[006]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [00000000410b3041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000000]
pmintenset[0000000000000001]
CPU[007]
pmuserenr [0000000000000000]
pmselr    [0000000000000000]
pmxevtyper[0000000088000008]
pmevtyper0[0000000088000008]
pmcntenset[0000000000000001]
pmcntenclr[0000000000000001]
pmcr      [00000000410b3041]
pmccfiltr [0000000000000000]
pmovsset  [0000000000000000]
pmintenset[0000000000000001]

@cyring
Copy link
Author

cyring commented Dec 28, 2023

Misc issues related to Kernel and PMU

  • Performance counters reset itself [1]
  • PMU registers get reset for 410c with Debian 144 [2]
  • nohlt [3]

@cyring
Copy link
Author

cyring commented Dec 28, 2023

nohlt is doing the trick. I now have a better control on PMU and Instructions Retired events are showing up into CoreFreq

2023-12-28-192922_642x410_scrot

@cyring
Copy link
Author

cyring commented Dec 30, 2023

dtc -I dtb -O dts rk3588-orangepi-5-plus.dtb > rk3588.dt
                cpu@0 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a55";
                        reg = <0x00>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x212>;
                        clocks = <0x0e 0x00>;
                        operating-points-v2 = <0x0f>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x8000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x80>;
                        d-cache-size = <0x8000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x80>;
                        next-level-cache = <0x11>;
                        #cooling-cells = <0x02>;
                        dynamic-power-coefficient = <0x64>;
                        cpu-supply = <0x12>;
                        mem-supply = <0x12>;
                        phandle = <0x06>;
                };

                cpu@100 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a55";
                        reg = <0x100>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x212>;
                        clocks = <0x0e 0x00>;
                        operating-points-v2 = <0x0f>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x8000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x80>;
                        d-cache-size = <0x8000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x80>;
                        next-level-cache = <0x13>;
                        phandle = <0x07>;
                };

                cpu@200 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a55";
                        reg = <0x200>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x212>;
                        clocks = <0x0e 0x00>;
                        operating-points-v2 = <0x0f>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x8000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x80>;
                        d-cache-size = <0x8000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x80>;
                        next-level-cache = <0x14>;
                        phandle = <0x08>;
                };

                cpu@300 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a55";
                        reg = <0x300>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x212>;
                        clocks = <0x0e 0x00>;
                        operating-points-v2 = <0x0f>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x8000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x80>;
                        d-cache-size = <0x8000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x80>;
                        next-level-cache = <0x15>;
                        phandle = <0x09>;
                };

                cpu@400 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a76";
                        reg = <0x400>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x400>;
                        clocks = <0x0e 0x02>;
                        operating-points-v2 = <0x16>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x10000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x100>;
                        d-cache-size = <0x10000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x100>;
                        next-level-cache = <0x17>;
                        #cooling-cells = <0x02>;
                        dynamic-power-coefficient = <0x12c>;
                        cpu-supply = <0x18>;
                        mem-supply = <0x18>;
                        phandle = <0x0a>;
                };

                cpu@500 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a76";
                        reg = <0x500>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x400>;
                        clocks = <0x0e 0x02>;
                        operating-points-v2 = <0x16>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x10000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x100>;
                        d-cache-size = <0x10000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x100>;
                        next-level-cache = <0x19>;
                        phandle = <0x0b>;
                };

                cpu@600 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a76";
                        reg = <0x600>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x400>;
                        clocks = <0x0e 0x03>;
                        operating-points-v2 = <0x1a>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x10000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x100>;
                        d-cache-size = <0x10000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x100>;
                        next-level-cache = <0x1b>;
                        #cooling-cells = <0x02>;
                        dynamic-power-coefficient = <0x12c>;
                        cpu-supply = <0x1c>;
                        mem-supply = <0x1c>;
                        phandle = <0x0c>;
                };

                cpu@700 {
                        device_type = "cpu";
                        compatible = "arm,cortex-a76";
                        reg = <0x700>;
                        enable-method = "psci";
                        capacity-dmips-mhz = <0x400>;
                        clocks = <0x0e 0x03>;
                        operating-points-v2 = <0x1a>;
                        cpu-idle-states = <0x10>;
                        i-cache-size = <0x10000>;
                        i-cache-line-size = <0x40>;
                        i-cache-sets = <0x100>;
                        d-cache-size = <0x10000>;
                        d-cache-line-size = <0x40>;
                        d-cache-sets = <0x100>;
                        next-level-cache = <0x1d>;
                        phandle = <0x0d>;
                };

                idle-states {
                        entry-method = "psci";

                        cpu-sleep {
                                compatible = "arm,idle-state";
                                local-timer-stop;
                                arm,psci-suspend-param = <0x10000>;
                                entry-latency-us = <0x64>;
                                exit-latency-us = <0x78>;
                                min-residency-us = <0x3e8>;
                                phandle = <0x10>;
                        };
                };

2023-12-30-220103_637x390_scrot

@cyring
Copy link
Author

cyring commented Dec 30, 2023

psci_idle_init() -> psci_cpuidle_driver.probe = psci_cpuidle_probe() -> for_each_possible_cpu(cpu) {
    psci_idle_init_cpu()
    {
        enable_method = of_get_property(cpu_node, "enable-method", NULL);
        if (!enable_method || (strcmp(enable_method, "psci")))
                ret = -ENODEV;

        drv->name = "psci_idle";
        drv->states[0].enter = psci_enter_idle_state;
        drv->states[0].exit_latency = 1;
        drv->states[0].target_residency = 1;
        drv->states[0].power_usage = UINT_MAX;
        strcpy(drv->states[0].name, "WFI");
        strcpy(drv->states[0].desc, "ARM WFI");

        ret = dt_init_idle_driver(drv /* name = "psci_idle" */, psci_idle_state_match /* .compatible = "arm,idle-state" */, 1);
        if (ret <= 0)
                return ret ? : -ENODEV;

        ret = psci_cpu_init_idle(dev, drv, cpu, ret);

        ret = cpuidle_register(drv, NULL);
    }
}

@cyring
Copy link
Author

cyring commented Dec 30, 2023

static __cpuidle int psci_enter_idle_state(struct cpuidle_device *dev,
                                           struct cpuidle_driver *drv, int idx)
{
        u32 *state = __this_cpu_read(psci_cpuidle_data.psci_states);

        return CPU_PM_CPU_IDLE_ENTER_PARAM_RCU(psci_cpu_suspend_enter, idx, state[idx]);
}
int psci_cpu_suspend_enter(u32 state)
{
    if ( smc(PSCI_1_0_FN_PSCI_FEATURES /* = 0x84000000+10 */) & (0x1 << 1) ) {
        /* PSCI_1_0_EXT_POWER_STATE_TYPE_MASK (0x1 << 30) */

        psci_ops.cpu_suspend() -> cpu_suspend()
    }
    else
    {
        /* PSCI_0_2_POWER_STATE_TYPE_MASK (0x1 << 16) */

        cpu_suspend()
    }
}
cpu_suspend() -> __cpu_suspend_enter() -> " br	x8 " -> _cpu_resume -> cpu_do_resume

@cyring
Copy link
Author

cyring commented Dec 31, 2023

Comment on the disablement of PMU and AMU

#ifdef CONFIG_CPU_PM

SYM_FUNC_START(cpu_do_resume)

/*	reset_pmuserenr_el0 x0			** Disable PMU access from EL0 */
/*	reset_amuserenr_el0 x0			** Disable AMU access from EL0 */

	isb
	ret
SYM_FUNC_END(cpu_do_resume)
#endif
	.macro	reset_pmuserenr_el0, tmpreg
	mrs	\tmpreg, id_aa64dfr0_el1
	sbfx	\tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4
	cmp	\tmpreg, #1			// Skip if no PMU present
	b.lt	9000f
	msr	pmuserenr_el0, xzr		// Disable PMU access from EL0
9000:
	.endm
	.macro	reset_amuserenr_el0, tmpreg
	mrs	\tmpreg, id_aa64pfr0_el1	// Check ID_AA64PFR0_EL1
	ubfx	\tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4
	cbz	\tmpreg, .Lskip_\@		// Skip if no AMU present
	msr_s	SYS_AMUSERENR_EL0, xzr		// Disable AMU access from EL0
.Lskip_\@:
	.endm

@cyring
Copy link
Author

cyring commented Dec 31, 2023

  • Disable CONFIG_ARM_PMUV3 w/ kernel 6
  • Disable CONFIG_ARM_PMU w/ kernel 5
  • Disable CONFIG_PERF_EVENTS
  • Disable CONFIG_KVM
  • drivers/perf/arm_pmuv3.c
static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
{
	armv8pmu_stop(cpu_pmu);

	armv8pmu_start(cpu_pmu);

	return IRQ_HANDLED;
}
armv8pmu_stop() -> armv8pmu_pmcr_write() -> write_pmcr() -> write_sysreg() -> __write_sysreg() -> pmcr_el0

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment