[RFC PATCH v3 18/25] net: Migrate user pages into PISA for MSG_ZEROCOPY
From: Luigi Rizzo
Date: Thu Nov 19 2026 - 20:13:31 EST
Perform best-effort migration of user pages into PISA memory on
MSG_ZEROCOPY sends in zerocopy_fill_skb_from_iter() when the egress
device has the per-device attribute pisa_tx_zerocopy enabled. For TCP
over a device that also has pisa_tx_bufs enabled, pages that cannot be
migrated fall back to the regular copy path into PISA TX page_frags
without reporting SO_EE_CODE_ZEROCOPY_COPIED.
Signed-off-by: Luigi Rizzo <lrizzo@xxxxxxxxxx>
---
include/linux/pisa.h | 3 ++
include/net/pisa.h | 32 +++++++++++++--
mm/pisa-user.c | 8 ++++
net/core/datagram.c | 17 ++++++--
net/core/pisa-net.c | 95 ++++++++++++++++++++++++++++++++++++++++++--
net/core/skbuff.c | 3 +-
net/ipv4/tcp.c | 12 +++++-
7 files changed, 158 insertions(+), 12 deletions(-)
diff --git a/include/linux/pisa.h b/include/linux/pisa.h
index 84692852299f1..11e3ea9eb263d 100644
--- a/include/linux/pisa.h
+++ b/include/linux/pisa.h
@@ -95,6 +95,7 @@ bool __pisa_page_can_recycle(const struct page *page);
bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
size_t size, size_t offset,
enum dma_data_direction dir, bool pinned);
+bool pisa_page_tx_ok(const struct page *page);
static inline bool pisa_enabled(void)
{
@@ -264,6 +265,8 @@ static inline bool pisa_upgrade_user_pages(struct iov_iter *iter,
return false;
}
+static inline bool pisa_page_tx_ok(const struct page *page) { return false; }
+
#endif /* CONFIG_PISA */
#if defined(CONFIG_PISA) && defined(CONFIG_IOMMU_DMA)
diff --git a/include/net/pisa.h b/include/net/pisa.h
index 7fe2e7d5aa97e..c95e4ab1f344f 100644
--- a/include/net/pisa.h
+++ b/include/net/pisa.h
@@ -13,9 +13,9 @@ struct sk_buff *__sock_alloc_send_pskb(struct sock *sk, unsigned long header_len
gfp_t gfp);
/*
- * struct net_device caches the PISA enable flags for tx_bufs and tx_heads used
- * by the leaf device, refreshed every 256 jiffies. The helpers below decide
- * whether to use PISA memory for a given dst, sk, skb.
+ * struct net_device caches the PISA enable flags for tx_bufs, tx_heads and
+ * tx_zerocopy used by the leaf device, refreshed every 256 jiffies. The helpers
+ * below decide whether to use PISA memory for a given dst, sk, skb.
*/
#ifdef CONFIG_PISA
@@ -23,6 +23,7 @@ enum pisa_tx_flags {
PISA_TX_FLAG_BUFS = BIT(0),
PISA_TX_FLAG_HEADS = BIT(1),
PISA_TX_XFRM_OK = BIT(2),
+ PISA_TX_FLAG_ZC = BIT(3),
PISA_TX_TSTAMP_MASK = ~0xffU,
};
@@ -36,6 +37,10 @@ DECLARE_STATIC_KEY_FALSE(net_pisa_tx_skb_heads_key);
u32 pisa_tx_flags_refresh(struct net_device *dev, u32 now);
void *__pisa_skb_head_alloc_node(gfp_t flags, int node);
+bool __pisa_zc_iter_stop(struct sk_buff *skb, struct iov_iter *from,
+ size_t *length, const struct sock *sk,
+ struct page **pages, int max_pages,
+ unsigned int orig_len, size_t *start, ssize_t *copied);
/*
* May TX data routed through @dst use PISA pages? Those pages stay readable by
@@ -144,6 +149,18 @@ static inline struct page_frag *sk_page_frag_dst(struct sock *sk,
sk->sk_use_task_frag ? task_page_frag(dst) : sk_page_frag(sk);
}
+/* Hook for __zerocopy_fill_skb_from_iter() MSG_ZEROCOPY upgrade (@pisa_user_key). */
+static inline bool pisa_zc_iter_stop(struct sk_buff *skb, struct iov_iter *from,
+ size_t *length, const struct sock *sk,
+ struct page **pages, int max_pages,
+ unsigned int orig_len, size_t *start,
+ ssize_t *copied)
+{
+ return pisa_user_enabled() &&
+ __pisa_zc_iter_stop(skb, from, length, sk, pages, max_pages,
+ orig_len, start, copied);
+}
+
#else /* !CONFIG_PISA */
static inline void *pisa_skb_head_alloc_node(gfp_t flags, int node) { return NULL; }
@@ -174,5 +191,14 @@ static inline struct page_frag *sk_page_frag_dst(struct sock *sk,
return sk_page_frag(sk);
}
+static inline bool pisa_zc_iter_stop(struct sk_buff *skb, struct iov_iter *from,
+ size_t *length, const struct sock *sk,
+ struct page **pages, int max_pages,
+ unsigned int orig_len, size_t *start,
+ ssize_t *copied)
+{
+ return false;
+}
+
#endif /* CONFIG_PISA */
#endif /* _NET_PISA_H */
diff --git a/mm/pisa-user.c b/mm/pisa-user.c
index 5fb1ec7c45204..e4a4e1a606b24 100644
--- a/mm/pisa-user.c
+++ b/mm/pisa-user.c
@@ -439,6 +439,14 @@ bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages,
return true;
}
+/* Is @page, referenced by the caller, PISA memory that devices can read (usable for TX)? */
+bool pisa_page_tx_ok(const struct page *page)
+{
+ unsigned long pfn = page_to_pfn(page);
+
+ return is_pisa_pfn(pfn) && (READ_ONCE(pisa_meta_of_pfn(pfn)->prot) & IOMMU_READ);
+}
+
/* Migrate the single-page segments already added to @bio; the caller then extracts them again. */
bool __pisa_bio_migrate_pages(struct bio *bio, struct iov_iter *iter,
enum dma_data_direction dir)
diff --git a/net/core/datagram.c b/net/core/datagram.c
index ecc30a6f83023..c589a543c3166 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -62,6 +62,7 @@
#include <net/tcp_states.h>
#include <trace/events/skb.h>
#include <net/busy_poll.h>
+#include <net/pisa.h>
#include "devmem.h"
@@ -632,10 +633,11 @@ int skb_copy_datagram_from_iter_full(struct sk_buff *skb, int offset,
}
EXPORT_SYMBOL(skb_copy_datagram_from_iter_full);
-int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
- struct iov_iter *from, size_t length)
+static int __zerocopy_fill_skb_from_iter(struct sk_buff *skb, struct iov_iter *from,
+ size_t length, const struct sock *sk)
{
int frag = skb_shinfo(skb)->nr_frags;
+ unsigned int orig_len = skb->len;
if (!skb_frags_readable(skb))
return -EFAULT;
@@ -654,6 +656,9 @@ int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
MAX_SKB_FRAGS - frag, &start);
if (copied < 0)
return -EFAULT;
+ if (pisa_zc_iter_stop(skb, from, &length, sk, pages,
+ MAX_SKB_FRAGS - frag, orig_len, &start, &copied))
+ return copied;
length -= copied;
@@ -703,6 +708,12 @@ int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
return 0;
}
+int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
+ struct iov_iter *from, size_t length)
+{
+ return __zerocopy_fill_skb_from_iter(skb, from, length, NULL);
+}
+
static int
zerocopy_fill_skb_from_devmem(struct sk_buff *skb, struct iov_iter *from,
int length,
@@ -760,7 +771,7 @@ int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
else if (binding)
ret = zerocopy_fill_skb_from_devmem(skb, from, length, binding);
else
- ret = zerocopy_fill_skb_from_iter(skb, from, length);
+ ret = __zerocopy_fill_skb_from_iter(skb, from, length, sk);
truesize = skb->truesize - orig_size;
if (sk && sk->sk_type == SOCK_STREAM) {
diff --git a/net/core/pisa-net.c b/net/core/pisa-net.c
index d0a2c94419187..6422c472029a1 100644
--- a/net/core/pisa-net.c
+++ b/net/core/pisa-net.c
@@ -12,6 +12,8 @@
#include <linux/pisa.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/uio.h>
#include <net/hotdata.h>
#include <net/pisa.h>
@@ -98,9 +100,10 @@ void *__pisa_skb_head_alloc_node(gfp_t flags, int node)
}
/*
- * The sysfs attributes pisa_tx_bufs and pisa_tx_skb_heads live on the
- * device that does DMA: the closest ancestor of a leaf net_device that has a
- * dma_mask (dev->dev.parent, or further up behind bus glue such as virtio).
+ * The sysfs attributes pisa_tx_bufs, pisa_tx_skb_heads and pisa_tx_zerocopy
+ * live on the device that does DMA: the closest ancestor of a leaf net_device
+ * that has a dma_mask (dev->dev.parent, or further up behind bus glue such as
+ * virtio).
* A socket route (dst->dev) may instead point to an upper virtual net_device
* (bond, VLAN, ipvlan, etc.), whose leaves are its lower devices.
* Walking lower devices via netdev_walk_all_lower_dev_rcu() on every TX
@@ -108,6 +111,7 @@ void *__pisa_skb_head_alloc_node(gfp_t flags, int node)
* result on the net_device and refreshes it at most once every 256 jiffies:
* bit 0 (PISA_TX_FLAG_BUFS): pisa_tx_bufs enabled on lower device(s)
* bit 1 (PISA_TX_FLAG_HEADS): pisa_tx_skb_heads enabled on lower device(s)
+ * bit 3 (PISA_TX_FLAG_ZC): pisa_tx_zerocopy enabled on lower device(s)
* bits 31..8 (PISA_TX_TSTAMP_MASK): upper 24 bits of (u32)jiffies
*/
static u32 pisa_tx_flags(const struct net_device *dev)
@@ -122,6 +126,8 @@ static u32 pisa_tx_flags(const struct net_device *dev)
flags |= PISA_TX_FLAG_BUFS;
if (READ_ONCE(parent->pisa_tx_skb_heads))
flags |= PISA_TX_FLAG_HEADS;
+ if (READ_ONCE(parent->pisa_tx_zerocopy))
+ flags |= PISA_TX_FLAG_ZC;
}
return flags;
}
@@ -160,7 +166,8 @@ u32 pisa_tx_flags_refresh(struct net_device *dev, u32 now)
now |= pisa_tx_flags(dev);
} else {
struct netdev_nested_priv priv = {
- .data = (void *)(uintptr_t)(PISA_TX_FLAG_BUFS | PISA_TX_FLAG_HEADS),
+ .data = (void *)(uintptr_t)(PISA_TX_FLAG_BUFS | PISA_TX_FLAG_HEADS |
+ PISA_TX_FLAG_ZC),
};
rcu_read_lock();
@@ -173,3 +180,83 @@ u32 pisa_tx_flags_refresh(struct net_device *dev, u32 now)
return now;
}
EXPORT_SYMBOL(pisa_tx_flags_refresh);
+
+/*
+ * Upgrade user @pages to PISA memory and, for TCP over a pisa_tx_bufs device,
+ * trim non-PISA trailing pages from the batch.
+ *
+ * MSG_ZEROCOPY sends of @sk may migrate user pages into PISA memory when the
+ * egress device has pisa_tx_zerocopy set. *@tx_bufs tells whether it also has
+ * pisa_tx_bufs set, i.e. whether a copy would land in PISA memory.
+ */
+static bool pisa_zc_enabled(const struct sock *sk, bool *tx_bufs)
+{
+ const struct dst_entry *dst;
+
+ if (!sk || !sk->sk_use_task_frag)
+ return false;
+ guard(rcu)();
+ dst = rcu_dereference(sk->sk_dst_cache);
+ if (!__pisa_tx_gfp(dst, PISA_TX_FLAG_ZC))
+ return false;
+ *tx_bufs = use_pisa_tx_bufs(dst);
+ return true;
+}
+
+bool __pisa_zc_iter_stop(struct sk_buff *skb, struct iov_iter *from,
+ size_t *length, const struct sock *sk,
+ struct page **pages, int max_pages,
+ unsigned int orig_len, size_t *start, ssize_t *copied)
+{
+ unsigned int i, nr;
+ bool tx_bufs;
+ ssize_t keep;
+
+ if (!user_backed_iter(from) || !pisa_zc_enabled(sk, &tx_bufs))
+ return false;
+
+ /* Move user pages into PISA memory, then look them up again. */
+ if (pisa_upgrade_user_pages(from, pages, *copied, *start,
+ DMA_TO_DEVICE, false)) {
+ *copied = iov_iter_get_pages2(from, pages, *length, max_pages, start);
+ if (*copied <= 0) {
+ if (*copied < 0)
+ *copied = -EFAULT;
+ return true;
+ }
+ }
+
+ /*
+ * Only TCP plain MSG_ZEROCOPY ubufs can fall back to copy, and the copy
+ * only helps if it goes into PISA memory.
+ */
+ if (!tx_bufs || skb->sk ||
+ (skb_zcopy(skb) && skb_zcopy(skb)->ops != &msg_zerocopy_ubuf_ops))
+ return false;
+
+ /*
+ * Trim @from to the leading pages that are in PISA memory devices can
+ * read (an RX-only page would not be mapped for TX): drop the
+ * references to the others, give their bytes back to @from, and cap
+ * *@length so that the caller ends after attaching what is left.
+ */
+ nr = DIV_ROUND_UP(*start + *copied, PAGE_SIZE);
+ for (i = 0; i < nr && pisa_page_tx_ok(pages[i]); i++)
+ ;
+ if (i == nr)
+ return false;
+
+ keep = i ? i * PAGE_SIZE - *start : 0;
+ while (nr > i)
+ put_page(pages[--nr]);
+ iov_iter_revert(from, *copied - keep);
+ *length = keep;
+ if (keep) {
+ *copied = keep;
+ return false;
+ }
+
+ /* -EXDEV tells tcp_sendmsg_locked() to copy this chunk instead of zerocopy. */
+ *copied = (skb->len != orig_len) ? 0 : -EXDEV;
+ return true;
+}
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 4f487e0450d4b..be6b6f42ee77e 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -1942,7 +1942,8 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len,
binding);
- if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
+ if (err == -EFAULT ||
+ ((err == -EMSGSIZE || err == -EXDEV) && skb->len == orig_len)) {
u32 delta_truesize = skb->truesize - orig_truesize;
struct sock *save_sk = skb->sk;
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 83f8ed2aa9a7e..411714edb90bd 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1132,6 +1132,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
int process_backlog = 0;
int sockc_err = 0;
int zc = 0;
+ bool must_copy = false; /* copy the next chunk of a MSG_ZEROCOPY send */
long timeo;
flags = msg->msg_flags;
@@ -1293,7 +1294,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
if (copy > msg_data_left(msg))
copy = msg_data_left(msg);
- if (zc == 0) {
+ if (zc == 0 || unlikely(must_copy)) {
bool merge = true;
int i = skb_shinfo(skb)->nr_frags;
struct page_frag *pfrag = sk_page_frag_dst(sk, __sk_dst_get(sk));
@@ -1338,6 +1339,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
page_ref_inc(pfrag->page);
}
pfrag->offset += copy;
+ must_copy = false;
} else if (zc == MSG_ZEROCOPY) {
/* First append to a fragless skb builds initial
* pure zerocopy skb
@@ -1357,6 +1359,14 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
tcp_mark_push(tp, skb);
goto new_segment;
}
+ /*
+ * The next user page is not in PISA memory: copy this
+ * chunk, without reporting SO_EE_CODE_ZEROCOPY_COPIED.
+ */
+ if (unlikely(err == -EXDEV)) {
+ must_copy = true;
+ continue;
+ }
if (err < 0)
goto do_error;
copy = err;
--
2.56.1.809.ga0b61035b5-goog