[PATCH v2 7/9] refcount: Prepare for atomic_*_overflow() offsets

[Peter Zijlstra]

In order to prepare for atomic_*_overflow() not strictly being in the
[0,INT_MAX] range, add ATOMIC_OVERFLOW_OFFSET to correct for the 0
point.

This makes the uncommon refcount ops a little awkward, but allows for
faster common ops.

Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
 include/linux/atomic.h   |   16 ++++++++++++++++
 include/linux/refcount.h |   16 ++++++++++++----
 lib/refcount.c           |   10 +++++-----
 3 files changed, 33 insertions(+), 9 deletions(-)

--- a/include/linux/atomic.h
+++ b/include/linux/atomic.h
@@ -7,6 +7,14 @@
 #include <asm/atomic.h>
 #include <asm/barrier.h>
 
+#ifndef ATOMIC_OVERFLOW_OFFSET
+#define ATOMIC_OVERFLOW_OFFSET 0
+#endif
+
+#ifndef ATOMIC64_OVERFLOW_OFFSET
+#define ATOMIC64_OVERFLOW_OFFSET 0
+#endif
+
 /*
  * Relaxed variants of xchg, cmpxchg and some atomic operations.
  *
@@ -81,4 +89,12 @@
 #include <linux/atomic/atomic-long.h>
 #include <linux/atomic/atomic-instrumented.h>
 
+#ifndef ATOMIC_LONG_OVERFLOW_OFFSET
+#ifdef CONFIG_64BIT
+#define ATOMIC_LONG_OVERFLOW_OFFSET	ATOMIC64_OVERFLOW_OFFSET
+#else
+#define ATOMIC_LONG_OVERFLOW_OFFSET	ATOMIC_OVERFLOW_OFFSET
+#endif
+#endif
+
 #endif /* _LINUX_ATOMIC_H */
--- a/include/linux/refcount.h
+++ b/include/linux/refcount.h
@@ -133,7 +133,7 @@ void refcount_warn_saturate(refcount_t *
  */
 static inline void refcount_set(refcount_t *r, int n)
 {
-	atomic_set(&r->refs, n);
+	atomic_set(&r->refs, n - ATOMIC_OVERFLOW_OFFSET);
 }
 
 /**
@@ -144,18 +144,20 @@ static inline void refcount_set(refcount
  */
 static inline unsigned int refcount_read(const refcount_t *r)
 {
-	return atomic_read(&r->refs);
+	return atomic_read(&r->refs) + ATOMIC_OVERFLOW_OFFSET;
 }
 
 static inline __must_check bool __refcount_add_not_zero(int i, refcount_t *r, int *oldp)
 {
-	int old = refcount_read(r);
+	int old = atomic_read(&r->refs);
 
 	do {
-		if (!old)
+		if (old == -ATOMIC_OVERFLOW_OFFSET)
 			break;
 	} while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i));
 
+	old += ATOMIC_OVERFLOW_OFFSET;
+
 	if (oldp)
 		*oldp = old;
 
@@ -192,6 +194,8 @@ static inline void __refcount_add(int i,
 {
 	int old = atomic_fetch_add_relaxed(i, &r->refs);
 
+	old += ATOMIC_OVERFLOW_OFFSET;
+
 	if (oldp)
 		*oldp = old;
 
@@ -274,6 +278,8 @@ static inline __must_check bool __refcou
 {
 	int old = atomic_fetch_sub_release(i, &r->refs);
 
+	old += ATOMIC_OVERFLOW_OFFSET;
+
 	if (oldp)
 		*oldp = old;
 
@@ -343,6 +349,8 @@ static inline void __refcount_dec(refcou
 {
 	int old = atomic_fetch_sub_release(1, &r->refs);
 
+	old += ATOMIC_OVERFLOW_OFFSET;
+
 	if (oldp)
 		*oldp = old;
 
--- a/lib/refcount.c
+++ b/lib/refcount.c
@@ -13,7 +13,7 @@
 void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t)
 {
 	int old = refcount_read(r);
-	refcount_set(r, REFCOUNT_SATURATED);
+	atomic_set(&r->refs, REFCOUNT_SATURATED);
 
 	/* racy; who cares */
 	if (old == 1 && t == REFCOUNT_ADD_OVF)
@@ -59,9 +59,9 @@ EXPORT_SYMBOL(refcount_warn_saturate);
  */
 bool refcount_dec_if_one(refcount_t *r)
 {
-	int val = 1;
+	int val = 1 - ATOMIC_OVERFLOW_OFFSET;
 
-	return atomic_try_cmpxchg_release(&r->refs, &val, 0);
+	return atomic_try_cmpxchg_release(&r->refs, &val, 0 - ATOMIC_OVERFLOW_OFFSET);
 }
 EXPORT_SYMBOL(refcount_dec_if_one);
 
@@ -85,11 +85,11 @@ bool refcount_dec_not_one(refcount_t *r)
 		if (unlikely(val - (REFCOUNT_SATURATED + REFCOUNT_SATURATED/2) < -REFCOUNT_SATURATED))
 			return true;
 
-		if (val == 1)
+		if (val == 1 - ATOMIC_OVERFLOW_OFFSET)
 			return false;
 
 		new = val - 1;
-		if (new > val) {
+		if (new + ATOMIC_OVERFLOW_OFFSET > val + ATOMIC_OVERFLOW_OFFSET) {
 			refcount_warn_saturate(r, REFCOUNT_SUB_UAF);
 			return true;
 		}