[PATCH] perf/x86/intel/bts: Fix confused ordering of PMU callbacks

[Alexander Shishkin]

The intel_bts driver is using a cpu-local 'started' variable to order
callbacks and PMIs and make sure that AUX transactions don't get messed
up. However, the ordering rules in regard to this variable is a complete
mess, which recently resulted in perf_fuzzer-triggered warnings and
panics.

The general ordering rule that is patch is enforcing is that this
cpu-local variable be set only when the cpu-local AUX transaction is
active; consequently, this variable is to be checked before the AUX
related bits can be touched.

Signed-off-by: Alexander Shishkin <alexander.shishkin@xxxxxxxxxxxxxxx>
---
 arch/x86/events/intel/bts.c | 119 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 96 insertions(+), 23 deletions(-)

diff --git a/arch/x86/events/intel/bts.c b/arch/x86/events/intel/bts.c
index 0a6e393a2e..13ce76e895 100644
--- a/arch/x86/events/intel/bts.c
+++ b/arch/x86/events/intel/bts.c
@@ -31,7 +31,18 @@
 struct bts_ctx {
 	struct perf_output_handle	handle;
 	struct debug_store		ds_back;
-	int				started;
+	local_t				pmi_pending;
+	int				state;
+};
+
+/* BTS context states: */
+enum {
+	/* no ongoing AUX transactions */
+	BTS_STATE_STOPPED = 0,
+	/* AUX transaction is on, BTS tracing is disabled */
+	BTS_STATE_INACTIVE,
+	/* AUX transaction is on, BTS tracing is running */
+	BTS_STATE_ACTIVE,
 };
 
 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
@@ -191,6 +202,9 @@ static void bts_update(struct bts_ctx *bts)
 		if (ds->bts_index >= ds->bts_absolute_maximum)
 			local_inc(&buf->lost);
 
+		if (ds->bts_index >= ds->bts_interrupt_threshold)
+			local_inc(&bts->pmi_pending);
+
 		/*
 		 * old and head are always in the same physical buffer, so we
 		 * can subtract them to get the data size.
@@ -204,6 +218,16 @@ static void bts_update(struct bts_ctx *bts)
 static int
 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
 
+/*
+ * Ordering PMU callbacks wrt themselves and the PMI is done by means
+ * of bts::state, which:
+ *  - is set when bts::handle::event is valid, that is, between
+ *    perf_aux_output_begin() and perf_aux_output_end();
+ *  - is zero otherwise;
+ *  - is ordered against bts::handle::event with a local barrier to
+ *    enforce CPU ordering.
+ */
+
 static void __bts_event_start(struct perf_event *event)
 {
 	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
@@ -221,10 +245,13 @@ static void __bts_event_start(struct perf_event *event)
 
 	/*
 	 * local barrier to make sure that ds configuration made it
-	 * before we enable BTS
+	 * before we enable BTS and bts::state goes ACTIVE
 	 */
 	wmb();
 
+	/* INACTIVE/STOPPED -> ACTIVE */
+	WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
+
 	intel_pmu_enable_bts(config);
 
 }
@@ -251,9 +278,6 @@ static void bts_event_start(struct perf_event *event, int flags)
 
 	__bts_event_start(event);
 
-	/* PMI handler: this counter is running and likely generating PMIs */
-	ACCESS_ONCE(bts->started) = 1;
-
 	return;
 
 fail_end_stop:
@@ -265,28 +289,36 @@ fail_stop:
 
 static void __bts_event_stop(struct perf_event *event)
 {
+	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
+
+	/* ACTIVE -> INACTIVE */
+	WRITE_ONCE(bts->state, BTS_STATE_INACTIVE);
+
 	/*
 	 * No extra synchronization is mandated by the documentation to have
 	 * BTS data stores globally visible.
 	 */
 	intel_pmu_disable_bts();
-
-	if (event->hw.state & PERF_HES_STOPPED)
-		return;
-
-	ACCESS_ONCE(event->hw.state) |= PERF_HES_STOPPED;
 }
 
 static void bts_event_stop(struct perf_event *event, int flags)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
-	struct bts_buffer *buf = perf_get_aux(&bts->handle);
+	struct bts_buffer *buf;
+
+	if (READ_ONCE(bts->state) == BTS_STATE_ACTIVE)
+		__bts_event_stop(event);
+
+	/*
+	 * order buf (handle::event load) against bts::state store;
+	 * matches wmb() in intel_bts_interrupt()
+	 */
+	mb();
 
-	/* PMI handler: don't restart this counter */
-	ACCESS_ONCE(bts->started) = 0;
+	buf = perf_get_aux(&bts->handle);
 
-	__bts_event_stop(event);
+	event->hw.state |= PERF_HES_STOPPED;
 
 	if (flags & PERF_EF_UPDATE) {
 		bts_update(bts);
@@ -298,6 +330,11 @@ static void bts_event_stop(struct perf_event *event, int flags)
 						   buf->nr_pages << PAGE_SHIFT);
 			perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
 					    !!local_xchg(&buf->lost, 0));
+			/*
+			 * no need to enforce ordering here as the PMI won't come after
+			 * __bts_event_stop()+mb() above
+			 */
+			WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
 		}
 
 		cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
@@ -311,7 +348,13 @@ void intel_bts_enable_local(void)
 {
 	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 
-	if (bts->handle.event && bts->started)
+	if (READ_ONCE(bts->state) == BTS_STATE_ACTIVE)
+		return;
+
+	/* matches wmb() ordering bts::state store against handle::event */
+	rmb();
+
+	if (bts->handle.event)
 		__bts_event_start(bts->handle.event);
 }
 
@@ -319,6 +362,12 @@ void intel_bts_disable_local(void)
 {
 	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 
+	if (READ_ONCE(bts->state) == BTS_STATE_INACTIVE)
+		return;
+
+	/* matches wmb() ordering bts::state store against handle::event */
+	rmb();
+
 	if (bts->handle.event)
 		__bts_event_stop(bts->handle.event);
 }
@@ -407,10 +456,25 @@ int intel_bts_interrupt(void)
 	struct perf_event *event = bts->handle.event;
 	struct bts_buffer *buf;
 	s64 old_head;
-	int err;
+	int err = -ENOSPC, handled = 0;
 
-	if (!event || !bts->started)
-		return 0;
+	if (local_read(&bts->pmi_pending)) {
+		handled = 1;
+		local_set(&bts->pmi_pending, 0);
+	}
+
+	/*
+	 * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
+	 * so we can only be INACTIVE or STOPPED
+	 */
+	if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
+		return handled;
+
+	/* matches wmb() ordering bts::state store against handle::event */
+	rmb();
+
+	if (!event)
+		return handled;
 
 	buf = perf_get_aux(&bts->handle);
 	/*
@@ -432,12 +496,21 @@ int intel_bts_interrupt(void)
 			    !!local_xchg(&buf->lost, 0));
 
 	buf = perf_aux_output_begin(&bts->handle, event);
-	if (!buf)
-		return 1;
+	if (buf)
+		err = bts_buffer_reset(buf, &bts->handle);
+
+	if (err) {
+		WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
 
-	err = bts_buffer_reset(buf, &bts->handle);
-	if (err)
-		perf_aux_output_end(&bts->handle, 0, false);
+		if (buf) {
+			/*
+			 * BTS_STATE_STOPPED should be visible before
+			 * cleared handle::event
+			 */
+			wmb();
+			perf_aux_output_end(&bts->handle, 0, false);
+		}
+	}
 
 	return 1;
 }
-- 
2.8.1