[PATCH v5 03/17] mm/mglru: make generation page counters atomic

From: Kairui Song via B4 Relay

Date: Wed Nov 18 2026 - 16:04:17 EST


From: Kairui Song <kasong@xxxxxxxxxxx>

No feature change, convert them to atomic so we can update them without
holding the LRU lock. A temporary underflow is fine: readers clamp a
negative value to zero, and the counters only need to be eventually
consistent.

Tested-by: Zhao Zhenzhuo <zhaozhengzhuo@xxxxxxxxxxxxx>
Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
---
include/linux/mm_inline.h | 6 ++----
include/linux/mmzone.h | 2 +-
mm/vmscan.c | 26 +++++++++++++-------------
mm/vmscan.h | 2 +-
4 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 2edb9beef6d3..46beded51c7d 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -265,11 +265,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli
VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);

if (old_gen >= 0)
- WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
- lrugen->nr_pages[old_gen][type][zone] - delta);
+ atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
if (new_gen >= 0)
- WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
- lrugen->nr_pages[new_gen][type][zone] + delta);
+ atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]);

/* addition */
if (old_gen < 0) {
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index c161d563ce8f..4c4aad95b0b6 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -554,7 +554,7 @@ struct lru_gen_folio {
/* the multi-gen LRU lists, lazily sorted on eviction */
struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the multi-gen LRU sizes, eventually consistent */
- long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
+ atomic_long_t nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
/* the exponential moving average of evicted+protected */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index fe1fe1439c7c..8d14831cdf83 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3350,8 +3350,7 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk)
continue;

walk->nr_pages[gen][type][zone] = 0;
- WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
- lrugen->nr_pages[gen][type][zone] + delta);
+ atomic_long_add(delta, &lrugen->nr_pages[gen][type][zone]);

if (lru_gen_is_active(lruvec, gen))
lru += LRU_ACTIVE;
@@ -3971,11 +3970,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
break;
}
flush_lru_batch(head, &batch_end, target_list);
-
- WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
- lrugen->nr_pages[old_gen][type][zone] - delta);
- WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone],
- lrugen->nr_pages[target_gen][type][zone] + delta);
+ atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]);
+ atomic_long_add(delta, &lrugen->nr_pages[target_gen][type][zone]);
if (!remaining)
return false;
}
@@ -4093,8 +4089,8 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness
for (type = 0; type < ANON_AND_FILE; type++) {
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
enum lru_list lru = type * LRU_INACTIVE_FILE;
- long delta = lrugen->nr_pages[prev][type][zone] -
- lrugen->nr_pages[next][type][zone];
+ long delta = atomic_long_read(&lrugen->nr_pages[prev][type][zone]) -
+ atomic_long_read(&lrugen->nr_pages[next][type][zone]);

if (!delta)
continue;
@@ -4625,8 +4621,13 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
child_lrugen = &child_lruvec->lrugen;
parent_lrugen = &parent_lruvec->lrugen;

+ /*
+ * The child is offline and the lockless updaters, which hold
+ * rcu_read_lock() over the update, avoid racing with reparenting,
+ * so the read/set/add operations below are safe.
+ */
for (gen = 0; gen < MAX_NR_GENS; gen++) {
- long nr_pages = child_lrugen->nr_pages[gen][type][zone];
+ long nr_pages = atomic_long_read(&child_lrugen->nr_pages[gen][type][zone]);
int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0;
int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0;

@@ -4634,9 +4635,8 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec
list_splice_tail_init(&child_lrugen->folios[gen][type][zone],
&parent_lrugen->folios[gen][type][zone]);

- WRITE_ONCE(child_lrugen->nr_pages[gen][type][zone], 0);
- WRITE_ONCE(parent_lrugen->nr_pages[gen][type][zone],
- parent_lrugen->nr_pages[gen][type][zone] + nr_pages);
+ atomic_long_set(&child_lrugen->nr_pages[gen][type][zone], 0);
+ atomic_long_add(nr_pages, &parent_lrugen->nr_pages[gen][type][zone]);

if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) {
__update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages);
diff --git a/mm/vmscan.h b/mm/vmscan.h
index c2cea36289f3..6551f0a6691a 100644
--- a/mm/vmscan.h
+++ b/mm/vmscan.h
@@ -126,7 +126,7 @@ static inline unsigned long lruvec_gen_size(struct lru_gen_folio *lrugen,
unsigned long size = 0;

for (int zone = 0; zone < max_zone; zone++)
- size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+ size += max(atomic_long_read(&lrugen->nr_pages[gen][type][zone]), 0L);

return size;
}

--
2.56.1