~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/xdp/xdp_umem.c

Version: ~ [ linux-5.18 ] ~ [ linux-5.17.9 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.41 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.117 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.195 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.244 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.280 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.315 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /* XDP user-space packet buffer
  3  * Copyright(c) 2018 Intel Corporation.
  4  */
  5 
  6 #include <linux/init.h>
  7 #include <linux/sched/mm.h>
  8 #include <linux/sched/signal.h>
  9 #include <linux/sched/task.h>
 10 #include <linux/uaccess.h>
 11 #include <linux/slab.h>
 12 #include <linux/bpf.h>
 13 #include <linux/mm.h>
 14 
 15 #include "xdp_umem.h"
 16 #include "xsk_queue.h"
 17 
 18 #define XDP_UMEM_MIN_CHUNK_SIZE 2048
 19 
 20 void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs)
 21 {
 22         unsigned long flags;
 23 
 24         spin_lock_irqsave(&umem->xsk_list_lock, flags);
 25         list_add_rcu(&xs->list, &umem->xsk_list);
 26         spin_unlock_irqrestore(&umem->xsk_list_lock, flags);
 27 }
 28 
 29 void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs)
 30 {
 31         unsigned long flags;
 32 
 33         if (xs->dev) {
 34                 spin_lock_irqsave(&umem->xsk_list_lock, flags);
 35                 list_del_rcu(&xs->list);
 36                 spin_unlock_irqrestore(&umem->xsk_list_lock, flags);
 37 
 38                 if (umem->zc)
 39                         synchronize_net();
 40         }
 41 }
 42 
 43 int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,
 44                         u32 queue_id, u16 flags)
 45 {
 46         bool force_zc, force_copy;
 47         struct netdev_bpf bpf;
 48         int err;
 49 
 50         force_zc = flags & XDP_ZEROCOPY;
 51         force_copy = flags & XDP_COPY;
 52 
 53         if (force_zc && force_copy)
 54                 return -EINVAL;
 55 
 56         if (force_copy)
 57                 return 0;
 58 
 59         dev_hold(dev);
 60 
 61         if (dev->netdev_ops->ndo_bpf && dev->netdev_ops->ndo_xsk_async_xmit) {
 62                 bpf.command = XDP_QUERY_XSK_UMEM;
 63 
 64                 rtnl_lock();
 65                 err = dev->netdev_ops->ndo_bpf(dev, &bpf);
 66                 rtnl_unlock();
 67 
 68                 if (err) {
 69                         dev_put(dev);
 70                         return force_zc ? -ENOTSUPP : 0;
 71                 }
 72 
 73                 bpf.command = XDP_SETUP_XSK_UMEM;
 74                 bpf.xsk.umem = umem;
 75                 bpf.xsk.queue_id = queue_id;
 76 
 77                 rtnl_lock();
 78                 err = dev->netdev_ops->ndo_bpf(dev, &bpf);
 79                 rtnl_unlock();
 80 
 81                 if (err) {
 82                         dev_put(dev);
 83                         return force_zc ? err : 0; /* fail or fallback */
 84                 }
 85 
 86                 umem->dev = dev;
 87                 umem->queue_id = queue_id;
 88                 umem->zc = true;
 89                 return 0;
 90         }
 91 
 92         dev_put(dev);
 93         return force_zc ? -ENOTSUPP : 0; /* fail or fallback */
 94 }
 95 
 96 static void xdp_umem_clear_dev(struct xdp_umem *umem)
 97 {
 98         struct netdev_bpf bpf;
 99         int err;
100 
101         if (umem->dev) {
102                 bpf.command = XDP_SETUP_XSK_UMEM;
103                 bpf.xsk.umem = NULL;
104                 bpf.xsk.queue_id = umem->queue_id;
105 
106                 rtnl_lock();
107                 err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf);
108                 rtnl_unlock();
109 
110                 if (err)
111                         WARN(1, "failed to disable umem!\n");
112 
113                 dev_put(umem->dev);
114                 umem->dev = NULL;
115         }
116 }
117 
118 static void xdp_umem_unpin_pages(struct xdp_umem *umem)
119 {
120         unsigned int i;
121 
122         for (i = 0; i < umem->npgs; i++) {
123                 struct page *page = umem->pgs[i];
124 
125                 set_page_dirty_lock(page);
126                 put_page(page);
127         }
128 
129         kfree(umem->pgs);
130         umem->pgs = NULL;
131 }
132 
133 static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
134 {
135         if (umem->user) {
136                 atomic_long_sub(umem->npgs, &umem->user->locked_vm);
137                 free_uid(umem->user);
138         }
139 }
140 
141 static void xdp_umem_release(struct xdp_umem *umem)
142 {
143         struct task_struct *task;
144         struct mm_struct *mm;
145 
146         xdp_umem_clear_dev(umem);
147 
148         if (umem->fq) {
149                 xskq_destroy(umem->fq);
150                 umem->fq = NULL;
151         }
152 
153         if (umem->cq) {
154                 xskq_destroy(umem->cq);
155                 umem->cq = NULL;
156         }
157 
158         xdp_umem_unpin_pages(umem);
159 
160         task = get_pid_task(umem->pid, PIDTYPE_PID);
161         put_pid(umem->pid);
162         if (!task)
163                 goto out;
164         mm = get_task_mm(task);
165         put_task_struct(task);
166         if (!mm)
167                 goto out;
168 
169         mmput(mm);
170         kfree(umem->pages);
171         umem->pages = NULL;
172 
173         xdp_umem_unaccount_pages(umem);
174 out:
175         kfree(umem);
176 }
177 
178 static void xdp_umem_release_deferred(struct work_struct *work)
179 {
180         struct xdp_umem *umem = container_of(work, struct xdp_umem, work);
181 
182         xdp_umem_release(umem);
183 }
184 
185 void xdp_get_umem(struct xdp_umem *umem)
186 {
187         refcount_inc(&umem->users);
188 }
189 
190 void xdp_put_umem(struct xdp_umem *umem)
191 {
192         if (!umem)
193                 return;
194 
195         if (refcount_dec_and_test(&umem->users)) {
196                 INIT_WORK(&umem->work, xdp_umem_release_deferred);
197                 schedule_work(&umem->work);
198         }
199 }
200 
201 static int xdp_umem_pin_pages(struct xdp_umem *umem)
202 {
203         unsigned int gup_flags = FOLL_WRITE;
204         long npgs;
205         int err;
206 
207         umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs),
208                             GFP_KERNEL | __GFP_NOWARN);
209         if (!umem->pgs)
210                 return -ENOMEM;
211 
212         down_write(&current->mm->mmap_sem);
213         npgs = get_user_pages(umem->address, umem->npgs,
214                               gup_flags, &umem->pgs[0], NULL);
215         up_write(&current->mm->mmap_sem);
216 
217         if (npgs != umem->npgs) {
218                 if (npgs >= 0) {
219                         umem->npgs = npgs;
220                         err = -ENOMEM;
221                         goto out_pin;
222                 }
223                 err = npgs;
224                 goto out_pgs;
225         }
226         return 0;
227 
228 out_pin:
229         xdp_umem_unpin_pages(umem);
230 out_pgs:
231         kfree(umem->pgs);
232         umem->pgs = NULL;
233         return err;
234 }
235 
236 static int xdp_umem_account_pages(struct xdp_umem *umem)
237 {
238         unsigned long lock_limit, new_npgs, old_npgs;
239 
240         if (capable(CAP_IPC_LOCK))
241                 return 0;
242 
243         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
244         umem->user = get_uid(current_user());
245 
246         do {
247                 old_npgs = atomic_long_read(&umem->user->locked_vm);
248                 new_npgs = old_npgs + umem->npgs;
249                 if (new_npgs > lock_limit) {
250                         free_uid(umem->user);
251                         umem->user = NULL;
252                         return -ENOBUFS;
253                 }
254         } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs,
255                                      new_npgs) != old_npgs);
256         return 0;
257 }
258 
259 static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
260 {
261         u32 chunk_size = mr->chunk_size, headroom = mr->headroom;
262         unsigned int chunks, chunks_per_page;
263         u64 addr = mr->addr, size = mr->len;
264         int size_chk, err, i;
265 
266         if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) {
267                 /* Strictly speaking we could support this, if:
268                  * - huge pages, or*
269                  * - using an IOMMU, or
270                  * - making sure the memory area is consecutive
271                  * but for now, we simply say "computer says no".
272                  */
273                 return -EINVAL;
274         }
275 
276         if (!is_power_of_2(chunk_size))
277                 return -EINVAL;
278 
279         if (!PAGE_ALIGNED(addr)) {
280                 /* Memory area has to be page size aligned. For
281                  * simplicity, this might change.
282                  */
283                 return -EINVAL;
284         }
285 
286         if ((addr + size) < addr)
287                 return -EINVAL;
288 
289         chunks = (unsigned int)div_u64(size, chunk_size);
290         if (chunks == 0)
291                 return -EINVAL;
292 
293         chunks_per_page = PAGE_SIZE / chunk_size;
294         if (chunks < chunks_per_page || chunks % chunks_per_page)
295                 return -EINVAL;
296 
297         headroom = ALIGN(headroom, 64);
298 
299         size_chk = chunk_size - headroom - XDP_PACKET_HEADROOM;
300         if (size_chk < 0)
301                 return -EINVAL;
302 
303         umem->pid = get_task_pid(current, PIDTYPE_PID);
304         umem->address = (unsigned long)addr;
305         umem->props.chunk_mask = ~((u64)chunk_size - 1);
306         umem->props.size = size;
307         umem->headroom = headroom;
308         umem->chunk_size_nohr = chunk_size - headroom;
309         umem->npgs = size / PAGE_SIZE;
310         umem->pgs = NULL;
311         umem->user = NULL;
312         INIT_LIST_HEAD(&umem->xsk_list);
313         spin_lock_init(&umem->xsk_list_lock);
314 
315         refcount_set(&umem->users, 1);
316 
317         err = xdp_umem_account_pages(umem);
318         if (err)
319                 goto out;
320 
321         err = xdp_umem_pin_pages(umem);
322         if (err)
323                 goto out_account;
324 
325         umem->pages = kcalloc(umem->npgs, sizeof(*umem->pages), GFP_KERNEL);
326         if (!umem->pages) {
327                 err = -ENOMEM;
328                 goto out_account;
329         }
330 
331         for (i = 0; i < umem->npgs; i++)
332                 umem->pages[i].addr = page_address(umem->pgs[i]);
333 
334         return 0;
335 
336 out_account:
337         xdp_umem_unaccount_pages(umem);
338 out:
339         put_pid(umem->pid);
340         return err;
341 }
342 
343 struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr)
344 {
345         struct xdp_umem *umem;
346         int err;
347 
348         umem = kzalloc(sizeof(*umem), GFP_KERNEL);
349         if (!umem)
350                 return ERR_PTR(-ENOMEM);
351 
352         err = xdp_umem_reg(umem, mr);
353         if (err) {
354                 kfree(umem);
355                 return ERR_PTR(err);
356         }
357 
358         return umem;
359 }
360 
361 bool xdp_umem_validate_queues(struct xdp_umem *umem)
362 {
363         return umem->fq && umem->cq;
364 }
365 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp