[PATCH 54/60] mm/migrate: route explicit NUMA migration to private nodes

From: Gregory Price via B4 Relay

Date: Fri Nov 20 2026 - 13:19:34 EST


From: "Gregory Price (Meta)" <gourry@xxxxxxxxxx>

move_pages(2) and migrate_pages(2) allocate the destination folio by
node id with __GFP_THISNODE. A private node is not on its own
__GFP_THISNODE (ZONELIST_NOFALLBACK) zonelist, so the allocation never
lands on it.

When the destination is a private node in N_MEMORY_MEMPOLICY, select the
node's private zonelist and a single-node mask in the syscall path, and
pass them to alloc_migration_target() via migration_target_control.

move_pages(2) now accepts such a destination rather than only common
memory, and still refuses other private nodes with -ENODEV, so
do_move_pages_to_node() only needs to check N_MEMORY_PRIVATE.
migrate_pages(2) keeps such nodes in its destination mask, and drops
other private nodes as it drops nodes outside the caller's cpuset.

The other users of alloc_migration_target() are unchanged.

HugeTLB folios still cannot move to a private node, since private nodes
hold no HugeTLB pages.

Signed-off-by: Gregory Price (Meta) <gourry@xxxxxxxxxx>
---
mm/internal.h | 1 +
mm/mempolicy.c | 19 +++++++++++++++----
mm/migrate.c | 13 ++++++++++++-
3 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/mm/internal.h b/mm/internal.h
index c83c100a5253..646efc2eacd1 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1309,6 +1309,7 @@ struct migration_target_control {
int nid; /* preferred node id */
nodemask_t *nmask;
gfp_t gfp_mask;
+ unsigned int alloc_flags; /* ALLOC_ZONELIST_PRIVATE or 0 */
enum migrate_reason reason;
};

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index cb5419cb7ee8..a7cbb05ac86c 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1340,7 +1340,7 @@ static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
static long migrate_to_node(struct mm_struct *mm, int source, int dest,
int flags)
{
- nodemask_t nmask;
+ nodemask_t nmask, dest_mask;
struct vm_area_struct *vma;
LIST_HEAD(pagelist);
long nr_failed;
@@ -1351,6 +1351,14 @@ static long migrate_to_node(struct mm_struct *mm, int source, int dest,
.reason = MR_SYSCALL,
};

+ /* Private placement is reachable only through the private zonelist. */
+ if (node_state(dest, N_MEMORY_MEMPOLICY) &&
+ node_state(dest, N_MEMORY_PRIVATE)) {
+ init_nodemask_of_node(&dest_mask, dest);
+ mtc.nmask = &dest_mask;
+ mtc.alloc_flags = ALLOC_ZONELIST_PRIVATE;
+ }
+
nodes_clear(nmask);
node_set(source, nmask);

@@ -1951,7 +1959,7 @@ static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
struct mm_struct *mm = NULL;
struct task_struct *task;
nodemask_t task_nodes;
- int err;
+ int err, nid;
nodemask_t *old;
nodemask_t *new;
NODEMASK_SCRATCH(scratch);
@@ -2001,8 +2009,11 @@ static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
}

task_nodes = cpuset_mems_allowed(current);
- /* Private memory is not a migration destination. */
- nodes_andnot(task_nodes, task_nodes, node_states[N_MEMORY_PRIVATE]);
+ /* Private nodes are destinations only if they permit userspace placement. */
+ for_each_node_state(nid, N_MEMORY_PRIVATE) {
+ if (!node_state(nid, N_MEMORY_MEMPOLICY))
+ node_clear(nid, task_nodes);
+ }
if (!nodes_and(*new, *new, task_nodes))
goto out_put;

diff --git a/mm/migrate.c b/mm/migrate.c
index 6791834f14fe..eda961d43df0 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2250,6 +2250,9 @@ struct folio *alloc_migration_target(struct folio *src, unsigned long private)
if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
gfp_mask |= __GFP_HIGHMEM;

+ if (mtc->alloc_flags)
+ return __folio_alloc_flags(gfp_mask, order, nid, mtc->nmask,
+ mtc->alloc_flags);
return __folio_alloc(gfp_mask, order, nid, mtc->nmask);
}

@@ -2267,6 +2270,7 @@ static int store_status(int __user *status, int start, int value, int nr)

static int do_move_pages_to_node(struct list_head *pagelist, int node)
{
+ nodemask_t nmask;
int err;
struct migration_target_control mtc = {
.nid = node,
@@ -2274,6 +2278,13 @@ static int do_move_pages_to_node(struct list_head *pagelist, int node)
.reason = MR_SYSCALL,
};

+ /* A private node is reachable only through its private zonelist. */
+ if (node_state(node, N_MEMORY_PRIVATE)) {
+ init_nodemask_of_node(&nmask, node);
+ mtc.nmask = &nmask;
+ mtc.alloc_flags = ALLOC_ZONELIST_PRIVATE;
+ }
+
err = migrate_pages(pagelist, alloc_migration_target, NULL,
(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
if (err)
@@ -2411,7 +2422,7 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
err = -ENODEV;
if (node < 0 || node >= MAX_NUMNODES)
goto out_flush;
- if (!node_state(node, N_MEMORY_COMMON))
+ if (!node_state(node, N_MEMORY_MEMPOLICY))
goto out_flush;

err = -EACCES;

--
2.56.1