~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/bpf/helpers.c

Version: ~ [ linux-5.18 ] ~ [ linux-5.17.9 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.41 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.117 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.195 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.244 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.280 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.315 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  2  *
  3  * This program is free software; you can redistribute it and/or
  4  * modify it under the terms of version 2 of the GNU General Public
  5  * License as published by the Free Software Foundation.
  6  *
  7  * This program is distributed in the hope that it will be useful, but
  8  * WITHOUT ANY WARRANTY; without even the implied warranty of
  9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 10  * General Public License for more details.
 11  */
 12 #include <linux/bpf.h>
 13 #include <linux/rcupdate.h>
 14 #include <linux/random.h>
 15 #include <linux/smp.h>
 16 #include <linux/topology.h>
 17 #include <linux/ktime.h>
 18 #include <linux/sched.h>
 19 #include <linux/uidgid.h>
 20 #include <linux/filter.h>
 21 
 22 /* If kernel subsystem is allowing eBPF programs to call this function,
 23  * inside its own verifier_ops->get_func_proto() callback it should return
 24  * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments
 25  *
 26  * Different map implementations will rely on rcu in map methods
 27  * lookup/update/delete, therefore eBPF programs must run under rcu lock
 28  * if program is allowed to access maps, so check rcu_read_lock_held in
 29  * all three functions.
 30  */
 31 BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
 32 {
 33         WARN_ON_ONCE(!rcu_read_lock_held());
 34         return (unsigned long) map->ops->map_lookup_elem(map, key);
 35 }
 36 
 37 const struct bpf_func_proto bpf_map_lookup_elem_proto = {
 38         .func           = bpf_map_lookup_elem,
 39         .gpl_only       = false,
 40         .pkt_access     = true,
 41         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 42         .arg1_type      = ARG_CONST_MAP_PTR,
 43         .arg2_type      = ARG_PTR_TO_MAP_KEY,
 44 };
 45 
 46 BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
 47            void *, value, u64, flags)
 48 {
 49         WARN_ON_ONCE(!rcu_read_lock_held());
 50         return map->ops->map_update_elem(map, key, value, flags);
 51 }
 52 
 53 const struct bpf_func_proto bpf_map_update_elem_proto = {
 54         .func           = bpf_map_update_elem,
 55         .gpl_only       = false,
 56         .pkt_access     = true,
 57         .ret_type       = RET_INTEGER,
 58         .arg1_type      = ARG_CONST_MAP_PTR,
 59         .arg2_type      = ARG_PTR_TO_MAP_KEY,
 60         .arg3_type      = ARG_PTR_TO_MAP_VALUE,
 61         .arg4_type      = ARG_ANYTHING,
 62 };
 63 
 64 BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
 65 {
 66         WARN_ON_ONCE(!rcu_read_lock_held());
 67         return map->ops->map_delete_elem(map, key);
 68 }
 69 
 70 const struct bpf_func_proto bpf_map_delete_elem_proto = {
 71         .func           = bpf_map_delete_elem,
 72         .gpl_only       = false,
 73         .pkt_access     = true,
 74         .ret_type       = RET_INTEGER,
 75         .arg1_type      = ARG_CONST_MAP_PTR,
 76         .arg2_type      = ARG_PTR_TO_MAP_KEY,
 77 };
 78 
 79 BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags)
 80 {
 81         return map->ops->map_push_elem(map, value, flags);
 82 }
 83 
 84 const struct bpf_func_proto bpf_map_push_elem_proto = {
 85         .func           = bpf_map_push_elem,
 86         .gpl_only       = false,
 87         .pkt_access     = true,
 88         .ret_type       = RET_INTEGER,
 89         .arg1_type      = ARG_CONST_MAP_PTR,
 90         .arg2_type      = ARG_PTR_TO_MAP_VALUE,
 91         .arg3_type      = ARG_ANYTHING,
 92 };
 93 
 94 BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
 95 {
 96         return map->ops->map_pop_elem(map, value);
 97 }
 98 
 99 const struct bpf_func_proto bpf_map_pop_elem_proto = {
100         .func           = bpf_map_pop_elem,
101         .gpl_only       = false,
102         .ret_type       = RET_INTEGER,
103         .arg1_type      = ARG_CONST_MAP_PTR,
104         .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
105 };
106 
107 BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
108 {
109         return map->ops->map_peek_elem(map, value);
110 }
111 
112 const struct bpf_func_proto bpf_map_peek_elem_proto = {
113         .func           = bpf_map_pop_elem,
114         .gpl_only       = false,
115         .ret_type       = RET_INTEGER,
116         .arg1_type      = ARG_CONST_MAP_PTR,
117         .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
118 };
119 
120 const struct bpf_func_proto bpf_get_prandom_u32_proto = {
121         .func           = bpf_user_rnd_u32,
122         .gpl_only       = false,
123         .ret_type       = RET_INTEGER,
124 };
125 
126 BPF_CALL_0(bpf_get_smp_processor_id)
127 {
128         return smp_processor_id();
129 }
130 
131 const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
132         .func           = bpf_get_smp_processor_id,
133         .gpl_only       = false,
134         .ret_type       = RET_INTEGER,
135 };
136 
137 BPF_CALL_0(bpf_get_numa_node_id)
138 {
139         return numa_node_id();
140 }
141 
142 const struct bpf_func_proto bpf_get_numa_node_id_proto = {
143         .func           = bpf_get_numa_node_id,
144         .gpl_only       = false,
145         .ret_type       = RET_INTEGER,
146 };
147 
148 BPF_CALL_0(bpf_ktime_get_ns)
149 {
150         /* NMI safe access to clock monotonic */
151         return ktime_get_mono_fast_ns();
152 }
153 
154 const struct bpf_func_proto bpf_ktime_get_ns_proto = {
155         .func           = bpf_ktime_get_ns,
156         .gpl_only       = true,
157         .ret_type       = RET_INTEGER,
158 };
159 
160 BPF_CALL_0(bpf_get_current_pid_tgid)
161 {
162         struct task_struct *task = current;
163 
164         if (unlikely(!task))
165                 return -EINVAL;
166 
167         return (u64) task->tgid << 32 | task->pid;
168 }
169 
170 const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {
171         .func           = bpf_get_current_pid_tgid,
172         .gpl_only       = false,
173         .ret_type       = RET_INTEGER,
174 };
175 
176 BPF_CALL_0(bpf_get_current_uid_gid)
177 {
178         struct task_struct *task = current;
179         kuid_t uid;
180         kgid_t gid;
181 
182         if (unlikely(!task))
183                 return -EINVAL;
184 
185         current_uid_gid(&uid, &gid);
186         return (u64) from_kgid(&init_user_ns, gid) << 32 |
187                      from_kuid(&init_user_ns, uid);
188 }
189 
190 const struct bpf_func_proto bpf_get_current_uid_gid_proto = {
191         .func           = bpf_get_current_uid_gid,
192         .gpl_only       = false,
193         .ret_type       = RET_INTEGER,
194 };
195 
196 BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)
197 {
198         struct task_struct *task = current;
199 
200         if (unlikely(!task))
201                 goto err_clear;
202 
203         strncpy(buf, task->comm, size);
204 
205         /* Verifier guarantees that size > 0. For task->comm exceeding
206          * size, guarantee that buf is %NUL-terminated. Unconditionally
207          * done here to save the size test.
208          */
209         buf[size - 1] = 0;
210         return 0;
211 err_clear:
212         memset(buf, 0, size);
213         return -EINVAL;
214 }
215 
216 const struct bpf_func_proto bpf_get_current_comm_proto = {
217         .func           = bpf_get_current_comm,
218         .gpl_only       = false,
219         .ret_type       = RET_INTEGER,
220         .arg1_type      = ARG_PTR_TO_UNINIT_MEM,
221         .arg2_type      = ARG_CONST_SIZE,
222 };
223 
224 #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK)
225 
226 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
227 {
228         arch_spinlock_t *l = (void *)lock;
229         union {
230                 __u32 val;
231                 arch_spinlock_t lock;
232         } u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED };
233 
234         compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");
235         BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));
236         BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));
237         arch_spin_lock(l);
238 }
239 
240 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
241 {
242         arch_spinlock_t *l = (void *)lock;
243 
244         arch_spin_unlock(l);
245 }
246 
247 #else
248 
249 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
250 {
251         atomic_t *l = (void *)lock;
252 
253         BUILD_BUG_ON(sizeof(*l) != sizeof(*lock));
254         do {
255                 atomic_cond_read_relaxed(l, !VAL);
256         } while (atomic_xchg(l, 1));
257 }
258 
259 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
260 {
261         atomic_t *l = (void *)lock;
262 
263         atomic_set_release(l, 0);
264 }
265 
266 #endif
267 
268 static DEFINE_PER_CPU(unsigned long, irqsave_flags);
269 
270 notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
271 {
272         unsigned long flags;
273 
274         local_irq_save(flags);
275         __bpf_spin_lock(lock);
276         __this_cpu_write(irqsave_flags, flags);
277         return 0;
278 }
279 
280 const struct bpf_func_proto bpf_spin_lock_proto = {
281         .func           = bpf_spin_lock,
282         .gpl_only       = false,
283         .ret_type       = RET_VOID,
284         .arg1_type      = ARG_PTR_TO_SPIN_LOCK,
285 };
286 
287 notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
288 {
289         unsigned long flags;
290 
291         flags = __this_cpu_read(irqsave_flags);
292         __bpf_spin_unlock(lock);
293         local_irq_restore(flags);
294         return 0;
295 }
296 
297 const struct bpf_func_proto bpf_spin_unlock_proto = {
298         .func           = bpf_spin_unlock,
299         .gpl_only       = false,
300         .ret_type       = RET_VOID,
301         .arg1_type      = ARG_PTR_TO_SPIN_LOCK,
302 };
303 
304 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
305                            bool lock_src)
306 {
307         struct bpf_spin_lock *lock;
308 
309         if (lock_src)
310                 lock = src + map->spin_lock_off;
311         else
312                 lock = dst + map->spin_lock_off;
313         preempt_disable();
314         ____bpf_spin_lock(lock);
315         copy_map_value(map, dst, src);
316         ____bpf_spin_unlock(lock);
317         preempt_enable();
318 }
319 
320 #ifdef CONFIG_CGROUPS
321 BPF_CALL_0(bpf_get_current_cgroup_id)
322 {
323         struct cgroup *cgrp = task_dfl_cgroup(current);
324 
325         return cgrp->kn->id.id;
326 }
327 
328 const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
329         .func           = bpf_get_current_cgroup_id,
330         .gpl_only       = false,
331         .ret_type       = RET_INTEGER,
332 };
333 
334 #ifdef CONFIG_CGROUP_BPF
335 DECLARE_PER_CPU(struct bpf_cgroup_storage*,
336                 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
337 
338 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
339 {
340         /* flags argument is not used now,
341          * but provides an ability to extend the API.
342          * verifier checks that its value is correct.
343          */
344         enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
345         struct bpf_cgroup_storage *storage;
346         void *ptr;
347 
348         storage = this_cpu_read(bpf_cgroup_storage[stype]);
349 
350         if (stype == BPF_CGROUP_STORAGE_SHARED)
351                 ptr = &READ_ONCE(storage->buf)->data[0];
352         else
353                 ptr = this_cpu_ptr(storage->percpu_buf);
354 
355         return (unsigned long)ptr;
356 }
357 
358 const struct bpf_func_proto bpf_get_local_storage_proto = {
359         .func           = bpf_get_local_storage,
360         .gpl_only       = false,
361         .ret_type       = RET_PTR_TO_MAP_VALUE,
362         .arg1_type      = ARG_CONST_MAP_PTR,
363         .arg2_type      = ARG_ANYTHING,
364 };
365 #endif
366 #endif
367 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp