~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/net/xdp.h

Version: ~ [ linux-5.16 ] ~ [ linux-5.15.13 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.90 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.170 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.224 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.261 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.296 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.298 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0-only */
  2 /* include/net/xdp.h
  3  *
  4  * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
  5  */
  6 #ifndef __LINUX_NET_XDP_H__
  7 #define __LINUX_NET_XDP_H__
  8 
  9 /**
 10  * DOC: XDP RX-queue information
 11  *
 12  * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
 13  * level RX-ring queues.  It is information that is specific to how
 14  * the driver have configured a given RX-ring queue.
 15  *
 16  * Each xdp_buff frame received in the driver carry a (pointer)
 17  * reference to this xdp_rxq_info structure.  This provides the XDP
 18  * data-path read-access to RX-info for both kernel and bpf-side
 19  * (limited subset).
 20  *
 21  * For now, direct access is only safe while running in NAPI/softirq
 22  * context.  Contents is read-mostly and must not be updated during
 23  * driver NAPI/softirq poll.
 24  *
 25  * The driver usage API is a register and unregister API.
 26  *
 27  * The struct is not directly tied to the XDP prog.  A new XDP prog
 28  * can be attached as long as it doesn't change the underlying
 29  * RX-ring.  If the RX-ring does change significantly, the NIC driver
 30  * naturally need to stop the RX-ring before purging and reallocating
 31  * memory.  In that process the driver MUST call unregistor (which
 32  * also apply for driver shutdown and unload).  The register API is
 33  * also mandatory during RX-ring setup.
 34  */
 35 
 36 enum xdp_mem_type {
 37         MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
 38         MEM_TYPE_PAGE_ORDER0,     /* Orig XDP full page model */
 39         MEM_TYPE_PAGE_POOL,
 40         MEM_TYPE_ZERO_COPY,
 41         MEM_TYPE_MAX,
 42 };
 43 
 44 /* XDP flags for ndo_xdp_xmit */
 45 #define XDP_XMIT_FLUSH          (1U << 0)       /* doorbell signal consumer */
 46 #define XDP_XMIT_FLAGS_MASK     XDP_XMIT_FLUSH
 47 
 48 struct xdp_mem_info {
 49         u32 type; /* enum xdp_mem_type, but known size type */
 50         u32 id;
 51 };
 52 
 53 struct page_pool;
 54 
 55 struct zero_copy_allocator {
 56         void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
 57 };
 58 
 59 struct xdp_rxq_info {
 60         struct net_device *dev;
 61         u32 queue_index;
 62         u32 reg_state;
 63         struct xdp_mem_info mem;
 64 } ____cacheline_aligned; /* perf critical, avoid false-sharing */
 65 
 66 struct xdp_buff {
 67         void *data;
 68         void *data_end;
 69         void *data_meta;
 70         void *data_hard_start;
 71         unsigned long handle;
 72         struct xdp_rxq_info *rxq;
 73 };
 74 
 75 struct xdp_frame {
 76         void *data;
 77         u16 len;
 78         u16 headroom;
 79         u16 metasize;
 80         /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
 81          * while mem info is valid on remote CPU.
 82          */
 83         struct xdp_mem_info mem;
 84         struct net_device *dev_rx; /* used by cpumap */
 85 };
 86 
 87 /* Clear kernel pointers in xdp_frame */
 88 static inline void xdp_scrub_frame(struct xdp_frame *frame)
 89 {
 90         frame->data = NULL;
 91         frame->dev_rx = NULL;
 92 }
 93 
 94 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
 95 
 96 /* Convert xdp_buff to xdp_frame */
 97 static inline
 98 struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
 99 {
100         struct xdp_frame *xdp_frame;
101         int metasize;
102         int headroom;
103 
104         if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
105                 return xdp_convert_zc_to_xdp_frame(xdp);
106 
107         /* Assure headroom is available for storing info */
108         headroom = xdp->data - xdp->data_hard_start;
109         metasize = xdp->data - xdp->data_meta;
110         metasize = metasize > 0 ? metasize : 0;
111         if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
112                 return NULL;
113 
114         /* Store info in top of packet */
115         xdp_frame = xdp->data_hard_start;
116 
117         xdp_frame->data = xdp->data;
118         xdp_frame->len  = xdp->data_end - xdp->data;
119         xdp_frame->headroom = headroom - sizeof(*xdp_frame);
120         xdp_frame->metasize = metasize;
121 
122         /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
123         xdp_frame->mem = xdp->rxq->mem;
124 
125         return xdp_frame;
126 }
127 
128 void xdp_return_frame(struct xdp_frame *xdpf);
129 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
130 void xdp_return_buff(struct xdp_buff *xdp);
131 
132 int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
133                      struct net_device *dev, u32 queue_index);
134 void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
135 void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
136 bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
137 int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
138                                enum xdp_mem_type type, void *allocator);
139 void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq);
140 
141 /* Drivers not supporting XDP metadata can use this helper, which
142  * rejects any room expansion for metadata as a result.
143  */
144 static __always_inline void
145 xdp_set_data_meta_invalid(struct xdp_buff *xdp)
146 {
147         xdp->data_meta = xdp->data + 1;
148 }
149 
150 static __always_inline bool
151 xdp_data_meta_unsupported(const struct xdp_buff *xdp)
152 {
153         return unlikely(xdp->data_meta > xdp->data);
154 }
155 
156 struct xdp_attachment_info {
157         struct bpf_prog *prog;
158         u32 flags;
159 };
160 
161 struct netdev_bpf;
162 int xdp_attachment_query(struct xdp_attachment_info *info,
163                          struct netdev_bpf *bpf);
164 bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
165                              struct netdev_bpf *bpf);
166 void xdp_attachment_setup(struct xdp_attachment_info *info,
167                           struct netdev_bpf *bpf);
168 
169 #endif /* __LINUX_NET_XDP_H__ */
170 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp