~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/ipv4/udp_tunnel_nic.c

Version: ~ [ linux-5.13-rc5 ] ~ [ linux-5.12.9 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.42 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.124 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.193 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.235 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.271 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.271 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 // Copyright (c) 2020 Facebook Inc.
  3 
  4 #include <linux/ethtool_netlink.h>
  5 #include <linux/netdevice.h>
  6 #include <linux/slab.h>
  7 #include <linux/types.h>
  8 #include <linux/workqueue.h>
  9 #include <net/udp_tunnel.h>
 10 #include <net/vxlan.h>
 11 
 12 enum udp_tunnel_nic_table_entry_flags {
 13         UDP_TUNNEL_NIC_ENTRY_ADD        = BIT(0),
 14         UDP_TUNNEL_NIC_ENTRY_DEL        = BIT(1),
 15         UDP_TUNNEL_NIC_ENTRY_OP_FAIL    = BIT(2),
 16         UDP_TUNNEL_NIC_ENTRY_FROZEN     = BIT(3),
 17 };
 18 
 19 struct udp_tunnel_nic_table_entry {
 20         __be16 port;
 21         u8 type;
 22         u8 use_cnt;
 23         u8 flags;
 24         u8 hw_priv;
 25 };
 26 
 27 /**
 28  * struct udp_tunnel_nic - UDP tunnel port offload state
 29  * @work:       async work for talking to hardware from process context
 30  * @dev:        netdev pointer
 31  * @need_sync:  at least one port start changed
 32  * @need_replay: space was freed, we need a replay of all ports
 33  * @work_pending: @work is currently scheduled
 34  * @n_tables:   number of tables under @entries
 35  * @missed:     bitmap of tables which overflown
 36  * @entries:    table of tables of ports currently offloaded
 37  */
 38 struct udp_tunnel_nic {
 39         struct work_struct work;
 40 
 41         struct net_device *dev;
 42 
 43         u8 need_sync:1;
 44         u8 need_replay:1;
 45         u8 work_pending:1;
 46 
 47         unsigned int n_tables;
 48         unsigned long missed;
 49         struct udp_tunnel_nic_table_entry **entries;
 50 };
 51 
 52 /* We ensure all work structs are done using driver state, but not the code.
 53  * We need a workqueue we can flush before module gets removed.
 54  */
 55 static struct workqueue_struct *udp_tunnel_nic_workqueue;
 56 
 57 static const char *udp_tunnel_nic_tunnel_type_name(unsigned int type)
 58 {
 59         switch (type) {
 60         case UDP_TUNNEL_TYPE_VXLAN:
 61                 return "vxlan";
 62         case UDP_TUNNEL_TYPE_GENEVE:
 63                 return "geneve";
 64         case UDP_TUNNEL_TYPE_VXLAN_GPE:
 65                 return "vxlan-gpe";
 66         default:
 67                 return "unknown";
 68         }
 69 }
 70 
 71 static bool
 72 udp_tunnel_nic_entry_is_free(struct udp_tunnel_nic_table_entry *entry)
 73 {
 74         return entry->use_cnt == 0 && !entry->flags;
 75 }
 76 
 77 static bool
 78 udp_tunnel_nic_entry_is_present(struct udp_tunnel_nic_table_entry *entry)
 79 {
 80         return entry->use_cnt && !(entry->flags & ~UDP_TUNNEL_NIC_ENTRY_FROZEN);
 81 }
 82 
 83 static bool
 84 udp_tunnel_nic_entry_is_frozen(struct udp_tunnel_nic_table_entry *entry)
 85 {
 86         return entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN;
 87 }
 88 
 89 static void
 90 udp_tunnel_nic_entry_freeze_used(struct udp_tunnel_nic_table_entry *entry)
 91 {
 92         if (!udp_tunnel_nic_entry_is_free(entry))
 93                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_FROZEN;
 94 }
 95 
 96 static void
 97 udp_tunnel_nic_entry_unfreeze(struct udp_tunnel_nic_table_entry *entry)
 98 {
 99         entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_FROZEN;
100 }
101 
102 static bool
103 udp_tunnel_nic_entry_is_queued(struct udp_tunnel_nic_table_entry *entry)
104 {
105         return entry->flags & (UDP_TUNNEL_NIC_ENTRY_ADD |
106                                UDP_TUNNEL_NIC_ENTRY_DEL);
107 }
108 
109 static void
110 udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn,
111                            struct udp_tunnel_nic_table_entry *entry,
112                            unsigned int flag)
113 {
114         entry->flags |= flag;
115         utn->need_sync = 1;
116 }
117 
118 static void
119 udp_tunnel_nic_ti_from_entry(struct udp_tunnel_nic_table_entry *entry,
120                              struct udp_tunnel_info *ti)
121 {
122         memset(ti, 0, sizeof(*ti));
123         ti->port = entry->port;
124         ti->type = entry->type;
125         ti->hw_priv = entry->hw_priv;
126 }
127 
128 static bool
129 udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn)
130 {
131         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
132         unsigned int i, j;
133 
134         for (i = 0; i < utn->n_tables; i++)
135                 for (j = 0; j < info->tables[i].n_entries; j++)
136                         if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
137                                 return false;
138         return true;
139 }
140 
141 static bool
142 udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
143 {
144         const struct udp_tunnel_nic_table_info *table;
145         unsigned int i, j;
146 
147         if (!utn->missed)
148                 return false;
149 
150         for (i = 0; i < utn->n_tables; i++) {
151                 table = &dev->udp_tunnel_nic_info->tables[i];
152                 if (!test_bit(i, &utn->missed))
153                         continue;
154 
155                 for (j = 0; j < table->n_entries; j++)
156                         if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))
157                                 return true;
158         }
159 
160         return false;
161 }
162 
163 static void
164 __udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table,
165                           unsigned int idx, struct udp_tunnel_info *ti)
166 {
167         struct udp_tunnel_nic_table_entry *entry;
168         struct udp_tunnel_nic *utn;
169 
170         utn = dev->udp_tunnel_nic;
171         entry = &utn->entries[table][idx];
172 
173         if (entry->use_cnt)
174                 udp_tunnel_nic_ti_from_entry(entry, ti);
175 }
176 
177 static void
178 __udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table,
179                                unsigned int idx, u8 priv)
180 {
181         dev->udp_tunnel_nic->entries[table][idx].hw_priv = priv;
182 }
183 
184 static void
185 udp_tunnel_nic_entry_update_done(struct udp_tunnel_nic_table_entry *entry,
186                                  int err)
187 {
188         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
189 
190         WARN_ON_ONCE(entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
191                      entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL);
192 
193         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD &&
194             (!err || (err == -EEXIST && dodgy)))
195                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_ADD;
196 
197         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL &&
198             (!err || (err == -ENOENT && dodgy)))
199                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_DEL;
200 
201         if (!err)
202                 entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
203         else
204                 entry->flags |= UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
205 }
206 
207 static void
208 udp_tunnel_nic_device_sync_one(struct net_device *dev,
209                                struct udp_tunnel_nic *utn,
210                                unsigned int table, unsigned int idx)
211 {
212         struct udp_tunnel_nic_table_entry *entry;
213         struct udp_tunnel_info ti;
214         int err;
215 
216         entry = &utn->entries[table][idx];
217         if (!udp_tunnel_nic_entry_is_queued(entry))
218                 return;
219 
220         udp_tunnel_nic_ti_from_entry(entry, &ti);
221         if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD)
222                 err = dev->udp_tunnel_nic_info->set_port(dev, table, idx, &ti);
223         else
224                 err = dev->udp_tunnel_nic_info->unset_port(dev, table, idx,
225                                                            &ti);
226         udp_tunnel_nic_entry_update_done(entry, err);
227 
228         if (err)
229                 netdev_warn(dev,
230                             "UDP tunnel port sync failed port %d type %s: %d\n",
231                             be16_to_cpu(entry->port),
232                             udp_tunnel_nic_tunnel_type_name(entry->type),
233                             err);
234 }
235 
236 static void
237 udp_tunnel_nic_device_sync_by_port(struct net_device *dev,
238                                    struct udp_tunnel_nic *utn)
239 {
240         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
241         unsigned int i, j;
242 
243         for (i = 0; i < utn->n_tables; i++)
244                 for (j = 0; j < info->tables[i].n_entries; j++)
245                         udp_tunnel_nic_device_sync_one(dev, utn, i, j);
246 }
247 
248 static void
249 udp_tunnel_nic_device_sync_by_table(struct net_device *dev,
250                                     struct udp_tunnel_nic *utn)
251 {
252         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
253         unsigned int i, j;
254         int err;
255 
256         for (i = 0; i < utn->n_tables; i++) {
257                 /* Find something that needs sync in this table */
258                 for (j = 0; j < info->tables[i].n_entries; j++)
259                         if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j]))
260                                 break;
261                 if (j == info->tables[i].n_entries)
262                         continue;
263 
264                 err = info->sync_table(dev, i);
265                 if (err)
266                         netdev_warn(dev, "UDP tunnel port sync failed for table %d: %d\n",
267                                     i, err);
268 
269                 for (j = 0; j < info->tables[i].n_entries; j++) {
270                         struct udp_tunnel_nic_table_entry *entry;
271 
272                         entry = &utn->entries[i][j];
273                         if (udp_tunnel_nic_entry_is_queued(entry))
274                                 udp_tunnel_nic_entry_update_done(entry, err);
275                 }
276         }
277 }
278 
279 static void
280 __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
281 {
282         if (!utn->need_sync)
283                 return;
284 
285         if (dev->udp_tunnel_nic_info->sync_table)
286                 udp_tunnel_nic_device_sync_by_table(dev, utn);
287         else
288                 udp_tunnel_nic_device_sync_by_port(dev, utn);
289 
290         utn->need_sync = 0;
291         /* Can't replay directly here, in case we come from the tunnel driver's
292          * notification - trying to replay may deadlock inside tunnel driver.
293          */
294         utn->need_replay = udp_tunnel_nic_should_replay(dev, utn);
295 }
296 
297 static void
298 udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
299 {
300         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
301         bool may_sleep;
302 
303         if (!utn->need_sync)
304                 return;
305 
306         /* Drivers which sleep in the callback need to update from
307          * the workqueue, if we come from the tunnel driver's notification.
308          */
309         may_sleep = info->flags & UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
310         if (!may_sleep)
311                 __udp_tunnel_nic_device_sync(dev, utn);
312         if (may_sleep || utn->need_replay) {
313                 queue_work(udp_tunnel_nic_workqueue, &utn->work);
314                 utn->work_pending = 1;
315         }
316 }
317 
318 static bool
319 udp_tunnel_nic_table_is_capable(const struct udp_tunnel_nic_table_info *table,
320                                 struct udp_tunnel_info *ti)
321 {
322         return table->tunnel_types & ti->type;
323 }
324 
325 static bool
326 udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn,
327                           struct udp_tunnel_info *ti)
328 {
329         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
330         unsigned int i;
331 
332         /* Special case IPv4-only NICs */
333         if (info->flags & UDP_TUNNEL_NIC_INFO_IPV4_ONLY &&
334             ti->sa_family != AF_INET)
335                 return false;
336 
337         for (i = 0; i < utn->n_tables; i++)
338                 if (udp_tunnel_nic_table_is_capable(&info->tables[i], ti))
339                         return true;
340         return false;
341 }
342 
343 static int
344 udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn,
345                              struct udp_tunnel_info *ti)
346 {
347         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
348         struct udp_tunnel_nic_table_entry *entry;
349         unsigned int i, j;
350 
351         for (i = 0; i < utn->n_tables; i++)
352                 for (j = 0; j < info->tables[i].n_entries; j++) {
353                         entry = &utn->entries[i][j];
354 
355                         if (!udp_tunnel_nic_entry_is_free(entry) &&
356                             entry->port == ti->port &&
357                             entry->type != ti->type) {
358                                 __set_bit(i, &utn->missed);
359                                 return true;
360                         }
361                 }
362         return false;
363 }
364 
365 static void
366 udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn,
367                          unsigned int table, unsigned int idx, int use_cnt_adj)
368 {
369         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
370         bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL;
371         unsigned int from, to;
372 
373         /* If not going from used to unused or vice versa - all done.
374          * For dodgy entries make sure we try to sync again (queue the entry).
375          */
376         entry->use_cnt += use_cnt_adj;
377         if (!dodgy && !entry->use_cnt == !(entry->use_cnt - use_cnt_adj))
378                 return;
379 
380         /* Cancel the op before it was sent to the device, if possible,
381          * otherwise we'd need to take special care to issue commands
382          * in the same order the ports arrived.
383          */
384         if (use_cnt_adj < 0) {
385                 from = UDP_TUNNEL_NIC_ENTRY_ADD;
386                 to = UDP_TUNNEL_NIC_ENTRY_DEL;
387         } else {
388                 from = UDP_TUNNEL_NIC_ENTRY_DEL;
389                 to = UDP_TUNNEL_NIC_ENTRY_ADD;
390         }
391 
392         if (entry->flags & from) {
393                 entry->flags &= ~from;
394                 if (!dodgy)
395                         return;
396         }
397 
398         udp_tunnel_nic_entry_queue(utn, entry, to);
399 }
400 
401 static bool
402 udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn,
403                              unsigned int table, unsigned int idx,
404                              struct udp_tunnel_info *ti, int use_cnt_adj)
405 {
406         struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];
407 
408         if (udp_tunnel_nic_entry_is_free(entry) ||
409             entry->port != ti->port ||
410             entry->type != ti->type)
411                 return false;
412 
413         if (udp_tunnel_nic_entry_is_frozen(entry))
414                 return true;
415 
416         udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj);
417         return true;
418 }
419 
420 /* Try to find existing matching entry and adjust its use count, instead of
421  * adding a new one. Returns true if entry was found. In case of delete the
422  * entry may have gotten removed in the process, in which case it will be
423  * queued for removal.
424  */
425 static bool
426 udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
427                             struct udp_tunnel_info *ti, int use_cnt_adj)
428 {
429         const struct udp_tunnel_nic_table_info *table;
430         unsigned int i, j;
431 
432         for (i = 0; i < utn->n_tables; i++) {
433                 table = &dev->udp_tunnel_nic_info->tables[i];
434                 if (!udp_tunnel_nic_table_is_capable(table, ti))
435                         continue;
436 
437                 for (j = 0; j < table->n_entries; j++)
438                         if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti,
439                                                          use_cnt_adj))
440                                 return true;
441         }
442 
443         return false;
444 }
445 
446 static bool
447 udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
448                             struct udp_tunnel_info *ti)
449 {
450         return udp_tunnel_nic_try_existing(dev, utn, ti, +1);
451 }
452 
453 static bool
454 udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn,
455                             struct udp_tunnel_info *ti)
456 {
457         return udp_tunnel_nic_try_existing(dev, utn, ti, -1);
458 }
459 
460 static bool
461 udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn,
462                        struct udp_tunnel_info *ti)
463 {
464         const struct udp_tunnel_nic_table_info *table;
465         unsigned int i, j;
466 
467         for (i = 0; i < utn->n_tables; i++) {
468                 table = &dev->udp_tunnel_nic_info->tables[i];
469                 if (!udp_tunnel_nic_table_is_capable(table, ti))
470                         continue;
471 
472                 for (j = 0; j < table->n_entries; j++) {
473                         struct udp_tunnel_nic_table_entry *entry;
474 
475                         entry = &utn->entries[i][j];
476                         if (!udp_tunnel_nic_entry_is_free(entry))
477                                 continue;
478 
479                         entry->port = ti->port;
480                         entry->type = ti->type;
481                         entry->use_cnt = 1;
482                         udp_tunnel_nic_entry_queue(utn, entry,
483                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
484                         return true;
485                 }
486 
487                 /* The different table may still fit this port in, but there
488                  * are no devices currently which have multiple tables accepting
489                  * the same tunnel type, and false positives are okay.
490                  */
491                 __set_bit(i, &utn->missed);
492         }
493 
494         return false;
495 }
496 
497 static void
498 __udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti)
499 {
500         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
501         struct udp_tunnel_nic *utn;
502 
503         utn = dev->udp_tunnel_nic;
504         if (!utn)
505                 return;
506         if (!netif_running(dev) && info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)
507                 return;
508         if (info->flags & UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN &&
509             ti->port == htons(IANA_VXLAN_UDP_PORT)) {
510                 if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
511                         netdev_warn(dev, "device assumes port 4789 will be used by vxlan tunnels\n");
512                 return;
513         }
514 
515         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
516                 return;
517 
518         /* It may happen that a tunnel of one type is removed and different
519          * tunnel type tries to reuse its port before the device was informed.
520          * Rely on utn->missed to re-add this port later.
521          */
522         if (udp_tunnel_nic_has_collision(dev, utn, ti))
523                 return;
524 
525         if (!udp_tunnel_nic_add_existing(dev, utn, ti))
526                 udp_tunnel_nic_add_new(dev, utn, ti);
527 
528         udp_tunnel_nic_device_sync(dev, utn);
529 }
530 
531 static void
532 __udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti)
533 {
534         struct udp_tunnel_nic *utn;
535 
536         utn = dev->udp_tunnel_nic;
537         if (!utn)
538                 return;
539 
540         if (!udp_tunnel_nic_is_capable(dev, utn, ti))
541                 return;
542 
543         udp_tunnel_nic_del_existing(dev, utn, ti);
544 
545         udp_tunnel_nic_device_sync(dev, utn);
546 }
547 
548 static void __udp_tunnel_nic_reset_ntf(struct net_device *dev)
549 {
550         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
551         struct udp_tunnel_nic *utn;
552         unsigned int i, j;
553 
554         ASSERT_RTNL();
555 
556         utn = dev->udp_tunnel_nic;
557         if (!utn)
558                 return;
559 
560         utn->need_sync = false;
561         for (i = 0; i < utn->n_tables; i++)
562                 for (j = 0; j < info->tables[i].n_entries; j++) {
563                         struct udp_tunnel_nic_table_entry *entry;
564 
565                         entry = &utn->entries[i][j];
566 
567                         entry->flags &= ~(UDP_TUNNEL_NIC_ENTRY_DEL |
568                                           UDP_TUNNEL_NIC_ENTRY_OP_FAIL);
569                         /* We don't release rtnl across ops */
570                         WARN_ON(entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN);
571                         if (!entry->use_cnt)
572                                 continue;
573 
574                         udp_tunnel_nic_entry_queue(utn, entry,
575                                                    UDP_TUNNEL_NIC_ENTRY_ADD);
576                 }
577 
578         __udp_tunnel_nic_device_sync(dev, utn);
579 }
580 
581 static size_t
582 __udp_tunnel_nic_dump_size(struct net_device *dev, unsigned int table)
583 {
584         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
585         struct udp_tunnel_nic *utn;
586         unsigned int j;
587         size_t size;
588 
589         utn = dev->udp_tunnel_nic;
590         if (!utn)
591                 return 0;
592 
593         size = 0;
594         for (j = 0; j < info->tables[table].n_entries; j++) {
595                 if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
596                         continue;
597 
598                 size += nla_total_size(0) +              /* _TABLE_ENTRY */
599                         nla_total_size(sizeof(__be16)) + /* _ENTRY_PORT */
600                         nla_total_size(sizeof(u32));     /* _ENTRY_TYPE */
601         }
602 
603         return size;
604 }
605 
606 static int
607 __udp_tunnel_nic_dump_write(struct net_device *dev, unsigned int table,
608                             struct sk_buff *skb)
609 {
610         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
611         struct udp_tunnel_nic *utn;
612         struct nlattr *nest;
613         unsigned int j;
614 
615         utn = dev->udp_tunnel_nic;
616         if (!utn)
617                 return 0;
618 
619         for (j = 0; j < info->tables[table].n_entries; j++) {
620                 if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))
621                         continue;
622 
623                 nest = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY);
624 
625                 if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT,
626                                  utn->entries[table][j].port) ||
627                     nla_put_u32(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_TYPE,
628                                 ilog2(utn->entries[table][j].type)))
629                         goto err_cancel;
630 
631                 nla_nest_end(skb, nest);
632         }
633 
634         return 0;
635 
636 err_cancel:
637         nla_nest_cancel(skb, nest);
638         return -EMSGSIZE;
639 }
640 
641 static const struct udp_tunnel_nic_ops __udp_tunnel_nic_ops = {
642         .get_port       = __udp_tunnel_nic_get_port,
643         .set_port_priv  = __udp_tunnel_nic_set_port_priv,
644         .add_port       = __udp_tunnel_nic_add_port,
645         .del_port       = __udp_tunnel_nic_del_port,
646         .reset_ntf      = __udp_tunnel_nic_reset_ntf,
647         .dump_size      = __udp_tunnel_nic_dump_size,
648         .dump_write     = __udp_tunnel_nic_dump_write,
649 };
650 
651 static void
652 udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn)
653 {
654         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
655         unsigned int i, j;
656 
657         for (i = 0; i < utn->n_tables; i++)
658                 for (j = 0; j < info->tables[i].n_entries; j++) {
659                         int adj_cnt = -utn->entries[i][j].use_cnt;
660 
661                         if (adj_cnt)
662                                 udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt);
663                 }
664 
665         __udp_tunnel_nic_device_sync(dev, utn);
666 
667         for (i = 0; i < utn->n_tables; i++)
668                 memset(utn->entries[i], 0, array_size(info->tables[i].n_entries,
669                                                       sizeof(**utn->entries)));
670         WARN_ON(utn->need_sync);
671         utn->need_replay = 0;
672 }
673 
674 static void
675 udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn)
676 {
677         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
678         unsigned int i, j;
679 
680         /* Freeze all the ports we are already tracking so that the replay
681          * does not double up the refcount.
682          */
683         for (i = 0; i < utn->n_tables; i++)
684                 for (j = 0; j < info->tables[i].n_entries; j++)
685                         udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]);
686         utn->missed = 0;
687         utn->need_replay = 0;
688 
689         udp_tunnel_get_rx_info(dev);
690 
691         for (i = 0; i < utn->n_tables; i++)
692                 for (j = 0; j < info->tables[i].n_entries; j++)
693                         udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]);
694 }
695 
696 static void udp_tunnel_nic_device_sync_work(struct work_struct *work)
697 {
698         struct udp_tunnel_nic *utn =
699                 container_of(work, struct udp_tunnel_nic, work);
700 
701         rtnl_lock();
702         utn->work_pending = 0;
703         __udp_tunnel_nic_device_sync(utn->dev, utn);
704 
705         if (utn->need_replay)
706                 udp_tunnel_nic_replay(utn->dev, utn);
707         rtnl_unlock();
708 }
709 
710 static struct udp_tunnel_nic *
711 udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info,
712                      unsigned int n_tables)
713 {
714         struct udp_tunnel_nic *utn;
715         unsigned int i;
716 
717         utn = kzalloc(sizeof(*utn), GFP_KERNEL);
718         if (!utn)
719                 return NULL;
720         utn->n_tables = n_tables;
721         INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work);
722 
723         utn->entries = kmalloc_array(n_tables, sizeof(void *), GFP_KERNEL);
724         if (!utn->entries)
725                 goto err_free_utn;
726 
727         for (i = 0; i < n_tables; i++) {
728                 utn->entries[i] = kcalloc(info->tables[i].n_entries,
729                                           sizeof(*utn->entries[i]), GFP_KERNEL);
730                 if (!utn->entries[i])
731                         goto err_free_prev_entries;
732         }
733 
734         return utn;
735 
736 err_free_prev_entries:
737         while (i--)
738                 kfree(utn->entries[i]);
739         kfree(utn->entries);
740 err_free_utn:
741         kfree(utn);
742         return NULL;
743 }
744 
745 static int udp_tunnel_nic_register(struct net_device *dev)
746 {
747         const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info;
748         struct udp_tunnel_nic *utn;
749         unsigned int n_tables, i;
750 
751         BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE <
752                      UDP_TUNNEL_NIC_MAX_TABLES);
753 
754         if (WARN_ON(!info->set_port != !info->unset_port) ||
755             WARN_ON(!info->set_port == !info->sync_table) ||
756             WARN_ON(!info->tables[0].n_entries))
757                 return -EINVAL;
758 
759         n_tables = 1;
760         for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) {
761                 if (!info->tables[i].n_entries)
762                         continue;
763 
764                 n_tables++;
765                 if (WARN_ON(!info->tables[i - 1].n_entries))
766                         return -EINVAL;
767         }
768 
769         utn = udp_tunnel_nic_alloc(info, n_tables);
770         if (!utn)
771                 return -ENOMEM;
772 
773         utn->dev = dev;
774         dev_hold(dev);
775         dev->udp_tunnel_nic = utn;
776 
777         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
778                 udp_tunnel_get_rx_info(dev);
779 
780         return 0;
781 }
782 
783 static void
784 udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
785 {
786         unsigned int i;
787 
788         /* Flush before we check work, so we don't waste time adding entries
789          * from the work which we will boot immediately.
790          */
791         udp_tunnel_nic_flush(dev, utn);
792 
793         /* Wait for the work to be done using the state, netdev core will
794          * retry unregister until we give up our reference on this device.
795          */
796         if (utn->work_pending)
797                 return;
798 
799         for (i = 0; i < utn->n_tables; i++)
800                 kfree(utn->entries[i]);
801         kfree(utn->entries);
802         kfree(utn);
803         dev->udp_tunnel_nic = NULL;
804         dev_put(dev);
805 }
806 
807 static int
808 udp_tunnel_nic_netdevice_event(struct notifier_block *unused,
809                                unsigned long event, void *ptr)
810 {
811         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
812         const struct udp_tunnel_nic_info *info;
813         struct udp_tunnel_nic *utn;
814 
815         info = dev->udp_tunnel_nic_info;
816         if (!info)
817                 return NOTIFY_DONE;
818 
819         if (event == NETDEV_REGISTER) {
820                 int err;
821 
822                 err = udp_tunnel_nic_register(dev);
823                 if (err)
824                         netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err);
825                 return notifier_from_errno(err);
826         }
827         /* All other events will need the udp_tunnel_nic state */
828         utn = dev->udp_tunnel_nic;
829         if (!utn)
830                 return NOTIFY_DONE;
831 
832         if (event == NETDEV_UNREGISTER) {
833                 udp_tunnel_nic_unregister(dev, utn);
834                 return NOTIFY_OK;
835         }
836 
837         /* All other events only matter if NIC has to be programmed open */
838         if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY))
839                 return NOTIFY_DONE;
840 
841         if (event == NETDEV_UP) {
842                 WARN_ON(!udp_tunnel_nic_is_empty(dev, utn));
843                 udp_tunnel_get_rx_info(dev);
844                 return NOTIFY_OK;
845         }
846         if (event == NETDEV_GOING_DOWN) {
847                 udp_tunnel_nic_flush(dev, utn);
848                 return NOTIFY_OK;
849         }
850 
851         return NOTIFY_DONE;
852 }
853 
854 static struct notifier_block udp_tunnel_nic_notifier_block __read_mostly = {
855         .notifier_call = udp_tunnel_nic_netdevice_event,
856 };
857 
858 static int __init udp_tunnel_nic_init_module(void)
859 {
860         int err;
861 
862         udp_tunnel_nic_workqueue = alloc_workqueue("udp_tunnel_nic", 0, 0);
863         if (!udp_tunnel_nic_workqueue)
864                 return -ENOMEM;
865 
866         rtnl_lock();
867         udp_tunnel_nic_ops = &__udp_tunnel_nic_ops;
868         rtnl_unlock();
869 
870         err = register_netdevice_notifier(&udp_tunnel_nic_notifier_block);
871         if (err)
872                 goto err_unset_ops;
873 
874         return 0;
875 
876 err_unset_ops:
877         rtnl_lock();
878         udp_tunnel_nic_ops = NULL;
879         rtnl_unlock();
880         destroy_workqueue(udp_tunnel_nic_workqueue);
881         return err;
882 }
883 late_initcall(udp_tunnel_nic_init_module);
884 
885 static void __exit udp_tunnel_nic_cleanup_module(void)
886 {
887         unregister_netdevice_notifier(&udp_tunnel_nic_notifier_block);
888 
889         rtnl_lock();
890         udp_tunnel_nic_ops = NULL;
891         rtnl_unlock();
892 
893         destroy_workqueue(udp_tunnel_nic_workqueue);
894 }
895 module_exit(udp_tunnel_nic_cleanup_module);
896 
897 MODULE_LICENSE("GPL");
898 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp