Re: [PATCH 2/4] perf, x86: Add Intel Nhm/Wsm/Snb load latency support

From: Stephane Eranian
Date: Fri Jul 22 2011 - 14:58:31 EST


Lin,

On Mon, Jul 4, 2011 at 1:02 AM, Lin Ming <ming.m.lin@xxxxxxxxx> wrote:
>
> Implements Intel memory load event for Nehalem/Westmere/SandyBridge.
>
> $ perf mem -t load record make -j8
>
> <building kernel ..., monitoring memory load opeartion>
>
> $ perf mem -t load report
>
> Memory load operation statistics
> ================================
> Â Â Â Â Â Â Â Â Â Â ÂL1-local: total latency= Â 28027, count= Â Â3355(avg=8)
> Â Â Â Â Â Â Â Â Â Â ÂL2-snoop: total latency= Â Â1430, count= Â Â Â29(avg=49)
> Â Â Â Â Â Â Â Â Â Â ÂL2-local: total latency= Â Â 124, count= Â Â Â 8(avg=15)
> Â Â Â Â Â Â L3-snoop, found M: total latency= Â Â 452, count= Â Â Â 4(avg=113)
> Â Â Â Â ÂL3-snoop, found no M: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> L3-snoop, no coherency actions: total latency= Â Â 875, count= Â Â Â18(avg=48)
> Â Â Â ÂL3-miss, snoop, shared: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> Â Â L3-miss, local, exclusive: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> Â Â Â ÂL3-miss, local, shared: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> Â ÂL3-miss, remote, exclusive: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> Â Â Â L3-miss, remote, shared: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> Â Â Â Â Â Â Â Â Â ÂUnknown L3: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> Â Â Â Â Â Â Â Â Â Â Â Â Â ÂIO: total latency= Â Â Â 0, count= Â Â Â 0(avg=0)
> Â Â Â Â Â Â Â Â Â Â ÂUncached: total latency= Â Â 464, count= Â Â Â30(avg=15)
>
> Signed-off-by: Lin Ming <ming.m.lin@xxxxxxxxx>
> ---
> Âarch/x86/include/asm/msr-index.h     Â|  Â2 +
> Âarch/x86/kernel/cpu/perf_event.c     Â|  10 ++++++
> Âarch/x86/kernel/cpu/perf_event_intel.c  Â|  20 +++++++++++-
> Âarch/x86/kernel/cpu/perf_event_intel_ds.c | Â 49 ++++++++++++++++++++++++++--
> Â4 files changed, 76 insertions(+), 5 deletions(-)
>
> diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
> index 485b4f1..da93a9d 100644
> --- a/arch/x86/include/asm/msr-index.h
> +++ b/arch/x86/include/asm/msr-index.h
> @@ -60,6 +60,8 @@
> Â#define MSR_IA32_DS_AREA Â Â Â Â Â Â Â 0x00000600
> Â#define MSR_IA32_PERF_CAPABILITIES Â Â 0x00000345
>
> +#define MSR_PEBS_LD_LAT_THRESHOLD Â Â Â0x000003f6
> +
> Â#define MSR_MTRRfix64K_00000 Â Â Â Â Â 0x00000250
> Â#define MSR_MTRRfix16K_80000 Â Â Â Â Â 0x00000258
> Â#define MSR_MTRRfix16K_A0000 Â Â Â Â Â 0x00000259
> diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
> index 3a0338b..ce380a7 100644
> --- a/arch/x86/kernel/cpu/perf_event.c
> +++ b/arch/x86/kernel/cpu/perf_event.c
> @@ -207,6 +207,9 @@ struct extra_reg {
> Â#define INTEL_EVENT_EXTRA_REG(event, msr, vm) Â\
> Â Â Â ÂEVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm)
> Â#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0)
> +#define INTEL_EVENT_EXTRA_REG2(event, msr, vm) Â Â\
> + Â Â Â EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
> + Â Â Â Â Â Â Â Â Â Â Â ARCH_PERFMON_EVENTSEL_UMASK, vm)
>
> Âunion perf_capabilities {
> Â Â Â Âstruct {
> @@ -406,6 +409,11 @@ static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
> Â Â Â Â Â Â Â Â Â Â Â Âcontinue;
> Â Â Â Â Â Â Â Âif (event->attr.config1 & ~er->valid_mask)
> Â Â Â Â Â Â Â Â Â Â Â Âreturn -EINVAL;
> +
> + Â Â Â Â Â Â Â /* The minimum value that may be programmed into MSR_PEBS_LD_LAT is 3 */
> + Â Â Â Â Â Â Â if (er->msr == MSR_PEBS_LD_LAT_THRESHOLD && event->attr.config1 < 3)
> + Â Â Â Â Â Â Â Â Â Â Â return -EINVAL;
> +
> Â Â Â Â Â Â Â Âevent->hw.extra_reg = er->msr;
> Â Â Â Â Â Â Â Âevent->hw.extra_config = event->attr.config1;
> Â Â Â Â Â Â Â Âbreak;
> @@ -617,6 +625,8 @@ static int x86_setup_perfctr(struct perf_event *event)
> Â Â Â Âif (config == -1LL)
> Â Â Â Â Â Â Â Âreturn -EINVAL;
>
> + Â Â Â x86_pmu_extra_regs(config, event);
> +
> Â Â Â Â/*
> Â Â Â Â * Branch tracing:
> Â Â Â Â */
> diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
> index 41178c8..dde9041 100644
> --- a/arch/x86/kernel/cpu/perf_event_intel.c
> +++ b/arch/x86/kernel/cpu/perf_event_intel.c
> @@ -1,6 +1,6 @@
> Â#ifdef CONFIG_CPU_SUP_INTEL
>
> -#define MAX_EXTRA_REGS 2
> +#define MAX_EXTRA_REGS 3
>
> Â/*
> Â* Per register state.
> @@ -89,6 +89,7 @@ static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
> Âstatic struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
> Â{
> Â Â Â ÂINTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
> + Â Â Â INTEL_EVENT_EXTRA_REG2(0x100b, MSR_PEBS_LD_LAT_THRESHOLD, 0xffff),
> Â Â Â ÂEVENT_EXTRA_END
> Â};
>
> @@ -123,10 +124,17 @@ static struct event_constraint intel_snb_event_constraints[] __read_mostly =
> Â Â Â ÂEVENT_CONSTRAINT_END
> Â};
>
> +static struct extra_reg intel_snb_extra_regs[] __read_mostly =
> +{
> + Â Â Â INTEL_EVENT_EXTRA_REG2(0x01cd, MSR_PEBS_LD_LAT_THRESHOLD, 0xffff),
> + Â Â Â EVENT_EXTRA_END
> +};
> +

As I described in arch/x86/kernel/perf_event.c, LD_LAT is NOT shared
between HT threads.
It seems the way your patch is written, LD_LAT will end up being
treated as OFFCORE_RSP
on NHM/WSM when HT is on, i.e., considered shared. You need to handle this in
intel_pmu_cpu_starting().

>
> Âstatic struct extra_reg intel_westmere_extra_regs[] __read_mostly =
> Â{
> Â Â Â ÂINTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
> Â Â Â ÂINTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff),
> + Â Â Â INTEL_EVENT_EXTRA_REG2(0x100b, MSR_PEBS_LD_LAT_THRESHOLD, 0xffff),
> Â Â Â ÂEVENT_EXTRA_END
> Â};
>
> @@ -1445,6 +1453,9 @@ static __init int intel_pmu_init(void)
> Â Â Â Â Â Â Â Â/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
> Â Â Â Â Â Â Â Âintel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1;
>
> + Â Â Â Â Â Â Â /* Memory load latency */
> + Â Â Â Â Â Â Â intel_perfmon_event_map[PERF_COUNT_HW_MEM_LOAD] = 0x100b;
> +
> Â Â Â Â Â Â Â Âif (ebx & 0x40) {
> Â Â Â Â Â Â Â Â Â Â Â Â/*
> Â Â Â Â Â Â Â Â Â Â Â Â * Erratum AAJ80 detected, we work it around by using
> @@ -1491,6 +1502,9 @@ static __init int intel_pmu_init(void)
> Â Â Â Â Â Â Â Â/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
> Â Â Â Â Â Â Â Âintel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1;
>
> + Â Â Â Â Â Â Â /* Memory load latency */
> + Â Â Â Â Â Â Â intel_perfmon_event_map[PERF_COUNT_HW_MEM_LOAD] = 0x100b;
> +
> Â Â Â Â Â Â Â Âpr_cont("Westmere events, ");
> Â Â Â Â Â Â Â Âbreak;
>
> @@ -1502,12 +1516,16 @@ static __init int intel_pmu_init(void)
>
> Â Â Â Â Â Â Â Âx86_pmu.event_constraints = intel_snb_event_constraints;
> Â Â Â Â Â Â Â Âx86_pmu.pebs_constraints = intel_snb_pebs_events;
> + Â Â Â Â Â Â Â x86_pmu.extra_regs = intel_snb_extra_regs;
>
> Â Â Â Â Â Â Â Â/* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
> Â Â Â Â Â Â Â Âintel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e;
> Â Â Â Â Â Â Â Â/* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
> Â Â Â Â Â Â Â Âintel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x18001b1;
>
> + Â Â Â Â Â Â Â /* Memory load latency */
> + Â Â Â Â Â Â Â intel_perfmon_event_map[PERF_COUNT_HW_MEM_LOAD] = 0x01cd;
> +
> Â Â Â Â Â Â Â Âpr_cont("SandyBridge events, ");
> Â Â Â Â Â Â Â Âbreak;
>
> diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
> index bab491b..d2d3155 100644
> --- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
> +++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
> @@ -1,5 +1,28 @@
> Â#ifdef CONFIG_CPU_SUP_INTEL
>
> +/* Indexed by Intel load latency data source encoding value */
> +
> +static u64 load_latency_data_source[] = {
> + Â Â Â MEM_LOAD_UNKNOWN | MEM_LOAD_TOGGLE, Â Â Â Â Â Â Â Â Â Â /* 0x00: Unknown L3 */
> + Â Â Â MEM_LOAD_L1 | MEM_LOAD_LOCAL, Â Â Â Â Â Â Â Â Â Â Â Â Â /* 0x01: L1-local */
> + Â Â Â MEM_LOAD_L2 | MEM_LOAD_SNOOP, Â Â Â Â Â Â Â Â Â Â Â Â Â /* 0x02: L2-snoop */
> + Â Â Â MEM_LOAD_L2 | MEM_LOAD_LOCAL, Â Â Â Â Â Â Â Â Â Â Â Â Â /* 0x03: L2-local */
> + Â Â Â MEM_LOAD_L3 | MEM_LOAD_SNOOP | MEM_LOAD_INVALID, Â Â Â Â/* 0x04: L3-snoop, no coherency actions */
> + Â Â Â MEM_LOAD_L3 | MEM_LOAD_SNOOP | MEM_LOAD_SHARED, Â Â Â Â /* 0x05: L3-snoop, found no M */
> + Â Â Â MEM_LOAD_L3 | MEM_LOAD_SNOOP | MEM_LOAD_MODIFIED, Â Â Â /* 0x06: L3-snoop, found M */
> + Â Â Â MEM_LOAD_RESERVED, Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â/* 0x07: reserved */
> + Â Â Â MEM_LOAD_RAM | MEM_LOAD_SNOOP | MEM_LOAD_SHARED, Â Â Â Â/* 0x08: L3-miss, snoop, shared */
> + Â Â Â MEM_LOAD_RESERVED, Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â/* 0x09: reserved */
> + Â Â Â MEM_LOAD_RAM | MEM_LOAD_LOCAL | MEM_LOAD_SHARED, Â Â Â Â/* 0x0A: L3-miss, local, shared */
> + Â Â Â MEM_LOAD_RAM | MEM_LOAD_REMOTE | MEM_LOAD_SHARED, Â Â Â /* 0x0B: L3-miss, remote, shared */
> + Â Â Â MEM_LOAD_RAM | MEM_LOAD_LOCAL | MEM_LOAD_EXCLUSIVE, Â Â /* 0x0C: L3-miss, local, exclusive */
> + Â Â Â MEM_LOAD_RAM | MEM_LOAD_REMOTE | MEM_LOAD_EXCLUSIVE, Â Â/* 0x0D: L3-miss, remote, exclusive */
> + Â Â Â MEM_LOAD_IO | MEM_LOAD_TOGGLE, Â Â Â Â Â Â Â Â Â Â Â Â Â/* 0x0E: IO */
> + Â Â Â MEM_LOAD_UNCACHED | MEM_LOAD_TOGGLE, Â Â Â Â Â Â Â Â Â Â/* 0x0F: Uncached */
> +};
> +
> +#define LOAD_LATENCY_DATA_SOURCE_MASK Â0x0FULL
> +
> Â/* The maximal number of PEBS events: */
> Â#define MAX_PEBS_EVENTS Â Â Â Â Â Â Â Â4
>
> @@ -454,6 +477,8 @@ static void intel_pmu_pebs_enable(struct perf_event *event)
> Â Â Â Âhwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
>
> Â Â Â Âcpuc->pebs_enabled |= 1ULL << hwc->idx;
> + Â Â Â if (hwc->extra_reg == MSR_PEBS_LD_LAT_THRESHOLD)
> + Â Â Â Â Â Â Â cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
> Â Â Â ÂWARN_ON_ONCE(cpuc->enabled);
>
> Â Â Â Âif (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
> @@ -466,6 +491,8 @@ static void intel_pmu_pebs_disable(struct perf_event *event)
> Â Â Â Âstruct hw_perf_event *hwc = &event->hw;
>
> Â Â Â Âcpuc->pebs_enabled &= ~(1ULL << hwc->idx);
> + Â Â Â if (hwc->extra_reg == MSR_PEBS_LD_LAT_THRESHOLD)
> + Â Â Â Â Â Â Â cpuc->pebs_enabled &= ~(1ULL << (hwc->idx + 32));

Something I don't understand here. How is the precise_ip mode selected
when you use LD_LAT? You need PEBS but there is still that off-by-1
error on the RIP. You'd like to user to be able to choose whether or not
to applyÂthe in-kernel correction. Yet, you've abstracted PEBS-LLÂin such
a wayÂthat the user is not even aware the kernel is enablingÂPEBS internally.
Something's wrong here.
That matters if you extract the load/stores addresses. It does not if
you just look at the latency distribution but that's too limiting in my mind.

>
> Â Â Â Âif (cpuc->enabled)
> Â Â Â Â Â Â Â Âwrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
>
> @@ -582,13 +609,13 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
> Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â struct pt_regs *iregs, void *__pebs)
> Â{
> Â Â Â Â/*
> - Â Â Â Â* We cast to pebs_record_core since that is a subset of
> - Â Â Â Â* both formats and we don't use the other fields in this
> - Â Â Â Â* routine.
> + Â Â Â Â* We cast to pebs_record_nhm to get the load latency data
> + Â Â Â Â* if extra_reg MSR_PEBS_LD_LAT_THRESHOLD used
> Â Â Â Â */
> - Â Â Â struct pebs_record_core *pebs = __pebs;
> + Â Â Â struct pebs_record_nhm *pebs = __pebs;
> Â Â Â Âstruct perf_sample_data data;
> Â Â Â Âstruct pt_regs regs;
> + Â Â Â u64 sample_type;
>
> Â Â Â Âif (!intel_pmu_save_and_restart(event))
> Â Â Â Â Â Â Â Âreturn;
> @@ -596,6 +623,20 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
> Â Â Â Âperf_sample_data_init(&data, 0);
> Â Â Â Âdata.period = event->hw.last_period;
>
> + Â Â Â if (event->attr.config == PERF_COUNT_HW_MEM_LOAD) {
> + Â Â Â Â Â Â Â sample_type = event->attr.sample_type;
> +
> + Â Â Â Â Â Â Â if (sample_type & PERF_SAMPLE_ADDR)
> + Â Â Â Â Â Â Â Â Â Â Â data.addr = pebs->dla;
> +
> + Â Â Â Â Â Â Â if (sample_type & PERF_SAMPLE_LATENCY)
> + Â Â Â Â Â Â Â Â Â Â Â data.latency = pebs->lat;
> +
> + Â Â Â Â Â Â Â if (sample_type & PERF_SAMPLE_EXTRA)
> + Â Â Â Â Â Â Â Â Â Â Â data.extra = load_latency_data_source[pebs->dse &
> + Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â LOAD_LATENCY_DATA_SOURCE_MASK];
> + Â Â Â }
> +
> Â Â Â Â/*
> Â Â Â Â * We use the interrupt regs as a base because the PEBS record
> Â Â Â Â * does not contain a full regs set, specifically it seems to
> --
> 1.7.5.1
>
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/