~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/smc/smc_core.c

Version: ~ [ linux-5.1-rc2 ] ~ [ linux-5.0.4 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.31 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.108 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.165 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.177 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.137 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
  4  *
  5  *  Basic Transport Functions exploiting Infiniband API
  6  *
  7  *  Copyright IBM Corp. 2016
  8  *
  9  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 10  */
 11 
 12 #include <linux/socket.h>
 13 #include <linux/if_vlan.h>
 14 #include <linux/random.h>
 15 #include <linux/workqueue.h>
 16 #include <net/tcp.h>
 17 #include <net/sock.h>
 18 #include <rdma/ib_verbs.h>
 19 #include <rdma/ib_cache.h>
 20 
 21 #include "smc.h"
 22 #include "smc_clc.h"
 23 #include "smc_core.h"
 24 #include "smc_ib.h"
 25 #include "smc_wr.h"
 26 #include "smc_llc.h"
 27 #include "smc_cdc.h"
 28 #include "smc_close.h"
 29 #include "smc_ism.h"
 30 
 31 #define SMC_LGR_NUM_INCR                256
 32 #define SMC_LGR_FREE_DELAY_SERV         (600 * HZ)
 33 #define SMC_LGR_FREE_DELAY_CLNT         (SMC_LGR_FREE_DELAY_SERV + 10 * HZ)
 34 #define SMC_LGR_FREE_DELAY_FAST         (8 * HZ)
 35 
 36 static struct smc_lgr_list smc_lgr_list = {     /* established link groups */
 37         .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock),
 38         .list = LIST_HEAD_INIT(smc_lgr_list.list),
 39         .num = 0,
 40 };
 41 
 42 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
 43                          struct smc_buf_desc *buf_desc);
 44 
 45 static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)
 46 {
 47         /* client link group creation always follows the server link group
 48          * creation. For client use a somewhat higher removal delay time,
 49          * otherwise there is a risk of out-of-sync link groups.
 50          */
 51         mod_delayed_work(system_wq, &lgr->free_work,
 52                          (!lgr->is_smcd && lgr->role == SMC_CLNT) ?
 53                          SMC_LGR_FREE_DELAY_CLNT : SMC_LGR_FREE_DELAY_SERV);
 54 }
 55 
 56 void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr)
 57 {
 58         mod_delayed_work(system_wq, &lgr->free_work, SMC_LGR_FREE_DELAY_FAST);
 59 }
 60 
 61 /* Register connection's alert token in our lookup structure.
 62  * To use rbtrees we have to implement our own insert core.
 63  * Requires @conns_lock
 64  * @smc         connection to register
 65  * Returns 0 on success, != otherwise.
 66  */
 67 static void smc_lgr_add_alert_token(struct smc_connection *conn)
 68 {
 69         struct rb_node **link, *parent = NULL;
 70         u32 token = conn->alert_token_local;
 71 
 72         link = &conn->lgr->conns_all.rb_node;
 73         while (*link) {
 74                 struct smc_connection *cur = rb_entry(*link,
 75                                         struct smc_connection, alert_node);
 76 
 77                 parent = *link;
 78                 if (cur->alert_token_local > token)
 79                         link = &parent->rb_left;
 80                 else
 81                         link = &parent->rb_right;
 82         }
 83         /* Put the new node there */
 84         rb_link_node(&conn->alert_node, parent, link);
 85         rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);
 86 }
 87 
 88 /* Register connection in link group by assigning an alert token
 89  * registered in a search tree.
 90  * Requires @conns_lock
 91  * Note that '' is a reserved value and not assigned.
 92  */
 93 static void smc_lgr_register_conn(struct smc_connection *conn)
 94 {
 95         struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
 96         static atomic_t nexttoken = ATOMIC_INIT(0);
 97 
 98         /* find a new alert_token_local value not yet used by some connection
 99          * in this link group
100          */
101         sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */
102         while (!conn->alert_token_local) {
103                 conn->alert_token_local = atomic_inc_return(&nexttoken);
104                 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))
105                         conn->alert_token_local = 0;
106         }
107         smc_lgr_add_alert_token(conn);
108         conn->lgr->conns_num++;
109 }
110 
111 /* Unregister connection and reset the alert token of the given connection<
112  */
113 static void __smc_lgr_unregister_conn(struct smc_connection *conn)
114 {
115         struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
116         struct smc_link_group *lgr = conn->lgr;
117 
118         rb_erase(&conn->alert_node, &lgr->conns_all);
119         lgr->conns_num--;
120         conn->alert_token_local = 0;
121         sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */
122 }
123 
124 /* Unregister connection from lgr
125  */
126 static void smc_lgr_unregister_conn(struct smc_connection *conn)
127 {
128         struct smc_link_group *lgr = conn->lgr;
129 
130         if (!lgr)
131                 return;
132         write_lock_bh(&lgr->conns_lock);
133         if (conn->alert_token_local) {
134                 __smc_lgr_unregister_conn(conn);
135         }
136         write_unlock_bh(&lgr->conns_lock);
137 }
138 
139 /* Send delete link, either as client to request the initiation
140  * of the DELETE LINK sequence from server; or as server to
141  * initiate the delete processing. See smc_llc_rx_delete_link().
142  */
143 static int smc_link_send_delete(struct smc_link *lnk)
144 {
145         if (lnk->state == SMC_LNK_ACTIVE &&
146             !smc_llc_send_delete_link(lnk, SMC_LLC_REQ, true)) {
147                 smc_llc_link_deleting(lnk);
148                 return 0;
149         }
150         return -ENOTCONN;
151 }
152 
153 static void smc_lgr_free(struct smc_link_group *lgr);
154 
155 static void smc_lgr_free_work(struct work_struct *work)
156 {
157         struct smc_link_group *lgr = container_of(to_delayed_work(work),
158                                                   struct smc_link_group,
159                                                   free_work);
160         bool conns;
161 
162         spin_lock_bh(&smc_lgr_list.lock);
163         read_lock_bh(&lgr->conns_lock);
164         conns = RB_EMPTY_ROOT(&lgr->conns_all);
165         read_unlock_bh(&lgr->conns_lock);
166         if (!conns) { /* number of lgr connections is no longer zero */
167                 spin_unlock_bh(&smc_lgr_list.lock);
168                 return;
169         }
170         if (!list_empty(&lgr->list))
171                 list_del_init(&lgr->list); /* remove from smc_lgr_list */
172         spin_unlock_bh(&smc_lgr_list.lock);
173 
174         if (!lgr->is_smcd && !lgr->terminating) {
175                 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
176 
177                 /* try to send del link msg, on error free lgr immediately */
178                 if (lnk->state == SMC_LNK_ACTIVE &&
179                     !smc_link_send_delete(lnk)) {
180                         /* reschedule in case we never receive a response */
181                         smc_lgr_schedule_free_work(lgr);
182                         return;
183                 }
184         }
185 
186         if (!delayed_work_pending(&lgr->free_work)) {
187                 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
188 
189                 if (!lgr->is_smcd && lnk->state != SMC_LNK_INACTIVE)
190                         smc_llc_link_inactive(lnk);
191                 if (lgr->is_smcd)
192                         smc_ism_signal_shutdown(lgr);
193                 smc_lgr_free(lgr);
194         }
195 }
196 
197 /* create a new SMC link group */
198 static int smc_lgr_create(struct smc_sock *smc, bool is_smcd,
199                           struct smc_ib_device *smcibdev, u8 ibport,
200                           char *peer_systemid, unsigned short vlan_id,
201                           struct smcd_dev *smcismdev, u64 peer_gid)
202 {
203         struct smc_link_group *lgr;
204         struct smc_link *lnk;
205         u8 rndvec[3];
206         int rc = 0;
207         int i;
208 
209         if (is_smcd && vlan_id) {
210                 rc = smc_ism_get_vlan(smcismdev, vlan_id);
211                 if (rc)
212                         goto out;
213         }
214 
215         lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);
216         if (!lgr) {
217                 rc = -ENOMEM;
218                 goto out;
219         }
220         lgr->is_smcd = is_smcd;
221         lgr->sync_err = 0;
222         lgr->vlan_id = vlan_id;
223         rwlock_init(&lgr->sndbufs_lock);
224         rwlock_init(&lgr->rmbs_lock);
225         rwlock_init(&lgr->conns_lock);
226         for (i = 0; i < SMC_RMBE_SIZES; i++) {
227                 INIT_LIST_HEAD(&lgr->sndbufs[i]);
228                 INIT_LIST_HEAD(&lgr->rmbs[i]);
229         }
230         smc_lgr_list.num += SMC_LGR_NUM_INCR;
231         memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE);
232         INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);
233         lgr->conns_all = RB_ROOT;
234         if (is_smcd) {
235                 /* SMC-D specific settings */
236                 lgr->peer_gid = peer_gid;
237                 lgr->smcd = smcismdev;
238         } else {
239                 /* SMC-R specific settings */
240                 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
241                 memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN);
242 
243                 lnk = &lgr->lnk[SMC_SINGLE_LINK];
244                 /* initialize link */
245                 lnk->state = SMC_LNK_ACTIVATING;
246                 lnk->link_id = SMC_SINGLE_LINK;
247                 lnk->smcibdev = smcibdev;
248                 lnk->ibport = ibport;
249                 lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu;
250                 if (!smcibdev->initialized)
251                         smc_ib_setup_per_ibdev(smcibdev);
252                 get_random_bytes(rndvec, sizeof(rndvec));
253                 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +
254                         (rndvec[2] << 16);
255                 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,
256                                           vlan_id, lnk->gid, &lnk->sgid_index);
257                 if (rc)
258                         goto free_lgr;
259                 rc = smc_llc_link_init(lnk);
260                 if (rc)
261                         goto free_lgr;
262                 rc = smc_wr_alloc_link_mem(lnk);
263                 if (rc)
264                         goto clear_llc_lnk;
265                 rc = smc_ib_create_protection_domain(lnk);
266                 if (rc)
267                         goto free_link_mem;
268                 rc = smc_ib_create_queue_pair(lnk);
269                 if (rc)
270                         goto dealloc_pd;
271                 rc = smc_wr_create_link(lnk);
272                 if (rc)
273                         goto destroy_qp;
274         }
275         smc->conn.lgr = lgr;
276         spin_lock_bh(&smc_lgr_list.lock);
277         list_add(&lgr->list, &smc_lgr_list.list);
278         spin_unlock_bh(&smc_lgr_list.lock);
279         return 0;
280 
281 destroy_qp:
282         smc_ib_destroy_queue_pair(lnk);
283 dealloc_pd:
284         smc_ib_dealloc_protection_domain(lnk);
285 free_link_mem:
286         smc_wr_free_link_mem(lnk);
287 clear_llc_lnk:
288         smc_llc_link_clear(lnk);
289 free_lgr:
290         kfree(lgr);
291 out:
292         return rc;
293 }
294 
295 static void smc_buf_unuse(struct smc_connection *conn,
296                           struct smc_link_group *lgr)
297 {
298         if (conn->sndbuf_desc)
299                 conn->sndbuf_desc->used = 0;
300         if (conn->rmb_desc) {
301                 if (!conn->rmb_desc->regerr) {
302                         if (!lgr->is_smcd) {
303                                 /* unregister rmb with peer */
304                                 smc_llc_do_delete_rkey(
305                                                 &lgr->lnk[SMC_SINGLE_LINK],
306                                                 conn->rmb_desc);
307                         }
308                         conn->rmb_desc->used = 0;
309                 } else {
310                         /* buf registration failed, reuse not possible */
311                         write_lock_bh(&lgr->rmbs_lock);
312                         list_del(&conn->rmb_desc->list);
313                         write_unlock_bh(&lgr->rmbs_lock);
314 
315                         smc_buf_free(lgr, true, conn->rmb_desc);
316                 }
317         }
318 }
319 
320 /* remove a finished connection from its link group */
321 void smc_conn_free(struct smc_connection *conn)
322 {
323         struct smc_link_group *lgr = conn->lgr;
324 
325         if (!lgr)
326                 return;
327         if (lgr->is_smcd) {
328                 smc_ism_unset_conn(conn);
329                 tasklet_kill(&conn->rx_tsklet);
330         } else {
331                 smc_cdc_tx_dismiss_slots(conn);
332         }
333         smc_lgr_unregister_conn(conn);
334         smc_buf_unuse(conn, lgr);               /* allow buffer reuse */
335         conn->lgr = NULL;
336 
337         if (!lgr->conns_num)
338                 smc_lgr_schedule_free_work(lgr);
339 }
340 
341 static void smc_link_clear(struct smc_link *lnk)
342 {
343         lnk->peer_qpn = 0;
344         smc_llc_link_clear(lnk);
345         smc_ib_modify_qp_reset(lnk);
346         smc_wr_free_link(lnk);
347         smc_ib_destroy_queue_pair(lnk);
348         smc_ib_dealloc_protection_domain(lnk);
349         smc_wr_free_link_mem(lnk);
350 }
351 
352 static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb,
353                           struct smc_buf_desc *buf_desc)
354 {
355         struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
356 
357         if (is_rmb) {
358                 if (buf_desc->mr_rx[SMC_SINGLE_LINK])
359                         smc_ib_put_memory_region(
360                                         buf_desc->mr_rx[SMC_SINGLE_LINK]);
361                 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
362                                     DMA_FROM_DEVICE);
363         } else {
364                 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
365                                     DMA_TO_DEVICE);
366         }
367         sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]);
368         if (buf_desc->pages)
369                 __free_pages(buf_desc->pages, buf_desc->order);
370         kfree(buf_desc);
371 }
372 
373 static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb,
374                           struct smc_buf_desc *buf_desc)
375 {
376         if (is_dmb) {
377                 /* restore original buf len */
378                 buf_desc->len += sizeof(struct smcd_cdc_msg);
379                 smc_ism_unregister_dmb(lgr->smcd, buf_desc);
380         } else {
381                 kfree(buf_desc->cpu_addr);
382         }
383         kfree(buf_desc);
384 }
385 
386 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
387                          struct smc_buf_desc *buf_desc)
388 {
389         if (lgr->is_smcd)
390                 smcd_buf_free(lgr, is_rmb, buf_desc);
391         else
392                 smcr_buf_free(lgr, is_rmb, buf_desc);
393 }
394 
395 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb)
396 {
397         struct smc_buf_desc *buf_desc, *bf_desc;
398         struct list_head *buf_list;
399         int i;
400 
401         for (i = 0; i < SMC_RMBE_SIZES; i++) {
402                 if (is_rmb)
403                         buf_list = &lgr->rmbs[i];
404                 else
405                         buf_list = &lgr->sndbufs[i];
406                 list_for_each_entry_safe(buf_desc, bf_desc, buf_list,
407                                          list) {
408                         list_del(&buf_desc->list);
409                         smc_buf_free(lgr, is_rmb, buf_desc);
410                 }
411         }
412 }
413 
414 static void smc_lgr_free_bufs(struct smc_link_group *lgr)
415 {
416         /* free send buffers */
417         __smc_lgr_free_bufs(lgr, false);
418         /* free rmbs */
419         __smc_lgr_free_bufs(lgr, true);
420 }
421 
422 /* remove a link group */
423 static void smc_lgr_free(struct smc_link_group *lgr)
424 {
425         smc_lgr_free_bufs(lgr);
426         if (lgr->is_smcd)
427                 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id);
428         else
429                 smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]);
430         kfree(lgr);
431 }
432 
433 void smc_lgr_forget(struct smc_link_group *lgr)
434 {
435         spin_lock_bh(&smc_lgr_list.lock);
436         /* do not use this link group for new connections */
437         if (!list_empty(&lgr->list))
438                 list_del_init(&lgr->list);
439         spin_unlock_bh(&smc_lgr_list.lock);
440 }
441 
442 /* terminate linkgroup abnormally */
443 static void __smc_lgr_terminate(struct smc_link_group *lgr)
444 {
445         struct smc_connection *conn;
446         struct smc_sock *smc;
447         struct rb_node *node;
448 
449         if (lgr->terminating)
450                 return; /* lgr already terminating */
451         lgr->terminating = 1;
452         if (!list_empty(&lgr->list)) /* forget lgr */
453                 list_del_init(&lgr->list);
454         if (!lgr->is_smcd)
455                 smc_llc_link_inactive(&lgr->lnk[SMC_SINGLE_LINK]);
456 
457         write_lock_bh(&lgr->conns_lock);
458         node = rb_first(&lgr->conns_all);
459         while (node) {
460                 conn = rb_entry(node, struct smc_connection, alert_node);
461                 smc = container_of(conn, struct smc_sock, conn);
462                 sock_hold(&smc->sk); /* sock_put in close work */
463                 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
464                 __smc_lgr_unregister_conn(conn);
465                 conn->lgr = NULL;
466                 write_unlock_bh(&lgr->conns_lock);
467                 if (!schedule_work(&conn->close_work))
468                         sock_put(&smc->sk);
469                 write_lock_bh(&lgr->conns_lock);
470                 node = rb_first(&lgr->conns_all);
471         }
472         write_unlock_bh(&lgr->conns_lock);
473         if (!lgr->is_smcd)
474                 wake_up(&lgr->lnk[SMC_SINGLE_LINK].wr_reg_wait);
475         smc_lgr_schedule_free_work(lgr);
476 }
477 
478 void smc_lgr_terminate(struct smc_link_group *lgr)
479 {
480         spin_lock_bh(&smc_lgr_list.lock);
481         __smc_lgr_terminate(lgr);
482         spin_unlock_bh(&smc_lgr_list.lock);
483 }
484 
485 /* Called when IB port is terminated */
486 void smc_port_terminate(struct smc_ib_device *smcibdev, u8 ibport)
487 {
488         struct smc_link_group *lgr, *l;
489 
490         spin_lock_bh(&smc_lgr_list.lock);
491         list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {
492                 if (!lgr->is_smcd &&
493                     lgr->lnk[SMC_SINGLE_LINK].smcibdev == smcibdev &&
494                     lgr->lnk[SMC_SINGLE_LINK].ibport == ibport)
495                         __smc_lgr_terminate(lgr);
496         }
497         spin_unlock_bh(&smc_lgr_list.lock);
498 }
499 
500 /* Called when SMC-D device is terminated or peer is lost */
501 void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid, unsigned short vlan)
502 {
503         struct smc_link_group *lgr, *l;
504         LIST_HEAD(lgr_free_list);
505 
506         /* run common cleanup function and build free list */
507         spin_lock_bh(&smc_lgr_list.lock);
508         list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) {
509                 if (lgr->is_smcd && lgr->smcd == dev &&
510                     (!peer_gid || lgr->peer_gid == peer_gid) &&
511                     (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) {
512                         __smc_lgr_terminate(lgr);
513                         list_move(&lgr->list, &lgr_free_list);
514                 }
515         }
516         spin_unlock_bh(&smc_lgr_list.lock);
517 
518         /* cancel the regular free workers and actually free lgrs */
519         list_for_each_entry_safe(lgr, l, &lgr_free_list, list) {
520                 list_del_init(&lgr->list);
521                 cancel_delayed_work_sync(&lgr->free_work);
522                 if (!peer_gid && vlan == VLAN_VID_MASK) /* dev terminated? */
523                         smc_ism_signal_shutdown(lgr);
524                 smc_lgr_free(lgr);
525         }
526 }
527 
528 /* Determine vlan of internal TCP socket.
529  * @vlan_id: address to store the determined vlan id into
530  */
531 int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id)
532 {
533         struct dst_entry *dst = sk_dst_get(clcsock->sk);
534         struct net_device *ndev;
535         int i, nest_lvl, rc = 0;
536 
537         *vlan_id = 0;
538         if (!dst) {
539                 rc = -ENOTCONN;
540                 goto out;
541         }
542         if (!dst->dev) {
543                 rc = -ENODEV;
544                 goto out_rel;
545         }
546 
547         ndev = dst->dev;
548         if (is_vlan_dev(ndev)) {
549                 *vlan_id = vlan_dev_vlan_id(ndev);
550                 goto out_rel;
551         }
552 
553         rtnl_lock();
554         nest_lvl = dev_get_nest_level(ndev);
555         for (i = 0; i < nest_lvl; i++) {
556                 struct list_head *lower = &ndev->adj_list.lower;
557 
558                 if (list_empty(lower))
559                         break;
560                 lower = lower->next;
561                 ndev = (struct net_device *)netdev_lower_get_next(ndev, &lower);
562                 if (is_vlan_dev(ndev)) {
563                         *vlan_id = vlan_dev_vlan_id(ndev);
564                         break;
565                 }
566         }
567         rtnl_unlock();
568 
569 out_rel:
570         dst_release(dst);
571 out:
572         return rc;
573 }
574 
575 static bool smcr_lgr_match(struct smc_link_group *lgr,
576                            struct smc_clc_msg_local *lcl,
577                            enum smc_lgr_role role, u32 clcqpn)
578 {
579         return !memcmp(lgr->peer_systemid, lcl->id_for_peer,
580                        SMC_SYSTEMID_LEN) &&
581                 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid,
582                         SMC_GID_SIZE) &&
583                 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac,
584                         sizeof(lcl->mac)) &&
585                 lgr->role == role &&
586                 (lgr->role == SMC_SERV ||
587                  lgr->lnk[SMC_SINGLE_LINK].peer_qpn == clcqpn);
588 }
589 
590 static bool smcd_lgr_match(struct smc_link_group *lgr,
591                            struct smcd_dev *smcismdev, u64 peer_gid)
592 {
593         return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev;
594 }
595 
596 /* create a new SMC connection (and a new link group if necessary) */
597 int smc_conn_create(struct smc_sock *smc, bool is_smcd, int srv_first_contact,
598                     struct smc_ib_device *smcibdev, u8 ibport, u32 clcqpn,
599                     struct smc_clc_msg_local *lcl, struct smcd_dev *smcd,
600                     u64 peer_gid)
601 {
602         struct smc_connection *conn = &smc->conn;
603         int local_contact = SMC_FIRST_CONTACT;
604         struct smc_link_group *lgr;
605         unsigned short vlan_id;
606         enum smc_lgr_role role;
607         int rc = 0;
608 
609         role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
610         rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id);
611         if (rc)
612                 return rc;
613 
614         if ((role == SMC_CLNT) && srv_first_contact)
615                 /* create new link group as well */
616                 goto create;
617 
618         /* determine if an existing link group can be reused */
619         spin_lock_bh(&smc_lgr_list.lock);
620         list_for_each_entry(lgr, &smc_lgr_list.list, list) {
621                 write_lock_bh(&lgr->conns_lock);
622                 if ((is_smcd ? smcd_lgr_match(lgr, smcd, peer_gid) :
623                      smcr_lgr_match(lgr, lcl, role, clcqpn)) &&
624                     !lgr->sync_err &&
625                     lgr->vlan_id == vlan_id &&
626                     (role == SMC_CLNT ||
627                      lgr->conns_num < SMC_RMBS_PER_LGR_MAX)) {
628                         /* link group found */
629                         local_contact = SMC_REUSE_CONTACT;
630                         conn->lgr = lgr;
631                         smc_lgr_register_conn(conn); /* add smc conn to lgr */
632                         if (delayed_work_pending(&lgr->free_work))
633                                 cancel_delayed_work(&lgr->free_work);
634                         write_unlock_bh(&lgr->conns_lock);
635                         break;
636                 }
637                 write_unlock_bh(&lgr->conns_lock);
638         }
639         spin_unlock_bh(&smc_lgr_list.lock);
640 
641         if (role == SMC_CLNT && !srv_first_contact &&
642             (local_contact == SMC_FIRST_CONTACT)) {
643                 /* Server reuses a link group, but Client wants to start
644                  * a new one
645                  * send out_of_sync decline, reason synchr. error
646                  */
647                 return -ENOLINK;
648         }
649 
650 create:
651         if (local_contact == SMC_FIRST_CONTACT) {
652                 rc = smc_lgr_create(smc, is_smcd, smcibdev, ibport,
653                                     lcl->id_for_peer, vlan_id, smcd, peer_gid);
654                 if (rc)
655                         goto out;
656                 smc_lgr_register_conn(conn); /* add smc conn to lgr */
657         }
658         conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
659         conn->local_tx_ctrl.len = SMC_WR_TX_SIZE;
660         conn->urg_state = SMC_URG_READ;
661         if (is_smcd) {
662                 conn->rx_off = sizeof(struct smcd_cdc_msg);
663                 smcd_cdc_rx_init(conn); /* init tasklet for this conn */
664         }
665 #ifndef KERNEL_HAS_ATOMIC64
666         spin_lock_init(&conn->acurs_lock);
667 #endif
668 
669 out:
670         return rc ? rc : local_contact;
671 }
672 
673 /* convert the RMB size into the compressed notation - minimum 16K.
674  * In contrast to plain ilog2, this rounds towards the next power of 2,
675  * so the socket application gets at least its desired sndbuf / rcvbuf size.
676  */
677 static u8 smc_compress_bufsize(int size)
678 {
679         u8 compressed;
680 
681         if (size <= SMC_BUF_MIN_SIZE)
682                 return 0;
683 
684         size = (size - 1) >> 14;
685         compressed = ilog2(size) + 1;
686         if (compressed >= SMC_RMBE_SIZES)
687                 compressed = SMC_RMBE_SIZES - 1;
688         return compressed;
689 }
690 
691 /* convert the RMB size from compressed notation into integer */
692 int smc_uncompress_bufsize(u8 compressed)
693 {
694         u32 size;
695 
696         size = 0x00000001 << (((int)compressed) + 14);
697         return (int)size;
698 }
699 
700 /* try to reuse a sndbuf or rmb description slot for a certain
701  * buffer size; if not available, return NULL
702  */
703 static struct smc_buf_desc *smc_buf_get_slot(int compressed_bufsize,
704                                              rwlock_t *lock,
705                                              struct list_head *buf_list)
706 {
707         struct smc_buf_desc *buf_slot;
708 
709         read_lock_bh(lock);
710         list_for_each_entry(buf_slot, buf_list, list) {
711                 if (cmpxchg(&buf_slot->used, 0, 1) == 0) {
712                         read_unlock_bh(lock);
713                         return buf_slot;
714                 }
715         }
716         read_unlock_bh(lock);
717         return NULL;
718 }
719 
720 /* one of the conditions for announcing a receiver's current window size is
721  * that it "results in a minimum increase in the window size of 10% of the
722  * receive buffer space" [RFC7609]
723  */
724 static inline int smc_rmb_wnd_update_limit(int rmbe_size)
725 {
726         return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
727 }
728 
729 static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr,
730                                                 bool is_rmb, int bufsize)
731 {
732         struct smc_buf_desc *buf_desc;
733         struct smc_link *lnk;
734         int rc;
735 
736         /* try to alloc a new buffer */
737         buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
738         if (!buf_desc)
739                 return ERR_PTR(-ENOMEM);
740 
741         buf_desc->order = get_order(bufsize);
742         buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN |
743                                       __GFP_NOMEMALLOC | __GFP_COMP |
744                                       __GFP_NORETRY | __GFP_ZERO,
745                                       buf_desc->order);
746         if (!buf_desc->pages) {
747                 kfree(buf_desc);
748                 return ERR_PTR(-EAGAIN);
749         }
750         buf_desc->cpu_addr = (void *)page_address(buf_desc->pages);
751 
752         /* build the sg table from the pages */
753         lnk = &lgr->lnk[SMC_SINGLE_LINK];
754         rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1,
755                             GFP_KERNEL);
756         if (rc) {
757                 smc_buf_free(lgr, is_rmb, buf_desc);
758                 return ERR_PTR(rc);
759         }
760         sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl,
761                    buf_desc->cpu_addr, bufsize);
762 
763         /* map sg table to DMA address */
764         rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc,
765                                is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
766         /* SMC protocol depends on mapping to one DMA address only */
767         if (rc != 1)  {
768                 smc_buf_free(lgr, is_rmb, buf_desc);
769                 return ERR_PTR(-EAGAIN);
770         }
771 
772         /* create a new memory region for the RMB */
773         if (is_rmb) {
774                 rc = smc_ib_get_memory_region(lnk->roce_pd,
775                                               IB_ACCESS_REMOTE_WRITE |
776                                               IB_ACCESS_LOCAL_WRITE,
777                                               buf_desc);
778                 if (rc) {
779                         smc_buf_free(lgr, is_rmb, buf_desc);
780                         return ERR_PTR(rc);
781                 }
782         }
783 
784         buf_desc->len = bufsize;
785         return buf_desc;
786 }
787 
788 #define SMCD_DMBE_SIZES         7 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */
789 
790 static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr,
791                                                 bool is_dmb, int bufsize)
792 {
793         struct smc_buf_desc *buf_desc;
794         int rc;
795 
796         if (smc_compress_bufsize(bufsize) > SMCD_DMBE_SIZES)
797                 return ERR_PTR(-EAGAIN);
798 
799         /* try to alloc a new DMB */
800         buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
801         if (!buf_desc)
802                 return ERR_PTR(-ENOMEM);
803         if (is_dmb) {
804                 rc = smc_ism_register_dmb(lgr, bufsize, buf_desc);
805                 if (rc) {
806                         kfree(buf_desc);
807                         return ERR_PTR(-EAGAIN);
808                 }
809                 buf_desc->pages = virt_to_page(buf_desc->cpu_addr);
810                 /* CDC header stored in buf. So, pretend it was smaller */
811                 buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg);
812         } else {
813                 buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL |
814                                              __GFP_NOWARN | __GFP_NORETRY |
815                                              __GFP_NOMEMALLOC);
816                 if (!buf_desc->cpu_addr) {
817                         kfree(buf_desc);
818                         return ERR_PTR(-EAGAIN);
819                 }
820                 buf_desc->len = bufsize;
821         }
822         return buf_desc;
823 }
824 
825 static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
826 {
827         struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM);
828         struct smc_connection *conn = &smc->conn;
829         struct smc_link_group *lgr = conn->lgr;
830         struct list_head *buf_list;
831         int bufsize, bufsize_short;
832         int sk_buf_size;
833         rwlock_t *lock;
834 
835         if (is_rmb)
836                 /* use socket recv buffer size (w/o overhead) as start value */
837                 sk_buf_size = smc->sk.sk_rcvbuf / 2;
838         else
839                 /* use socket send buffer size (w/o overhead) as start value */
840                 sk_buf_size = smc->sk.sk_sndbuf / 2;
841 
842         for (bufsize_short = smc_compress_bufsize(sk_buf_size);
843              bufsize_short >= 0; bufsize_short--) {
844 
845                 if (is_rmb) {
846                         lock = &lgr->rmbs_lock;
847                         buf_list = &lgr->rmbs[bufsize_short];
848                 } else {
849                         lock = &lgr->sndbufs_lock;
850                         buf_list = &lgr->sndbufs[bufsize_short];
851                 }
852                 bufsize = smc_uncompress_bufsize(bufsize_short);
853                 if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC)
854                         continue;
855 
856                 /* check for reusable slot in the link group */
857                 buf_desc = smc_buf_get_slot(bufsize_short, lock, buf_list);
858                 if (buf_desc) {
859                         memset(buf_desc->cpu_addr, 0, bufsize);
860                         break; /* found reusable slot */
861                 }
862 
863                 if (is_smcd)
864                         buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize);
865                 else
866                         buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize);
867 
868                 if (PTR_ERR(buf_desc) == -ENOMEM)
869                         break;
870                 if (IS_ERR(buf_desc))
871                         continue;
872 
873                 buf_desc->used = 1;
874                 write_lock_bh(lock);
875                 list_add(&buf_desc->list, buf_list);
876                 write_unlock_bh(lock);
877                 break; /* found */
878         }
879 
880         if (IS_ERR(buf_desc))
881                 return -ENOMEM;
882 
883         if (is_rmb) {
884                 conn->rmb_desc = buf_desc;
885                 conn->rmbe_size_short = bufsize_short;
886                 smc->sk.sk_rcvbuf = bufsize * 2;
887                 atomic_set(&conn->bytes_to_rcv, 0);
888                 conn->rmbe_update_limit =
889                         smc_rmb_wnd_update_limit(buf_desc->len);
890                 if (is_smcd)
891                         smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */
892         } else {
893                 conn->sndbuf_desc = buf_desc;
894                 smc->sk.sk_sndbuf = bufsize * 2;
895                 atomic_set(&conn->sndbuf_space, bufsize);
896         }
897         return 0;
898 }
899 
900 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn)
901 {
902         struct smc_link_group *lgr = conn->lgr;
903 
904         if (!conn->lgr || conn->lgr->is_smcd)
905                 return;
906         smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
907                                conn->sndbuf_desc, DMA_TO_DEVICE);
908 }
909 
910 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn)
911 {
912         struct smc_link_group *lgr = conn->lgr;
913 
914         if (!conn->lgr || conn->lgr->is_smcd)
915                 return;
916         smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
917                                   conn->sndbuf_desc, DMA_TO_DEVICE);
918 }
919 
920 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
921 {
922         struct smc_link_group *lgr = conn->lgr;
923 
924         if (!conn->lgr || conn->lgr->is_smcd)
925                 return;
926         smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
927                                conn->rmb_desc, DMA_FROM_DEVICE);
928 }
929 
930 void smc_rmb_sync_sg_for_device(struct smc_connection *conn)
931 {
932         struct smc_link_group *lgr = conn->lgr;
933 
934         if (!conn->lgr || conn->lgr->is_smcd)
935                 return;
936         smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
937                                   conn->rmb_desc, DMA_FROM_DEVICE);
938 }
939 
940 /* create the send and receive buffer for an SMC socket;
941  * receive buffers are called RMBs;
942  * (even though the SMC protocol allows more than one RMB-element per RMB,
943  * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
944  * extra RMB for every connection in a link group
945  */
946 int smc_buf_create(struct smc_sock *smc, bool is_smcd)
947 {
948         int rc;
949 
950         /* create send buffer */
951         rc = __smc_buf_create(smc, is_smcd, false);
952         if (rc)
953                 return rc;
954         /* create rmb */
955         rc = __smc_buf_create(smc, is_smcd, true);
956         if (rc)
957                 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc);
958         return rc;
959 }
960 
961 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
962 {
963         int i;
964 
965         for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {
966                 if (!test_and_set_bit(i, lgr->rtokens_used_mask))
967                         return i;
968         }
969         return -ENOSPC;
970 }
971 
972 /* add a new rtoken from peer */
973 int smc_rtoken_add(struct smc_link_group *lgr, __be64 nw_vaddr, __be32 nw_rkey)
974 {
975         u64 dma_addr = be64_to_cpu(nw_vaddr);
976         u32 rkey = ntohl(nw_rkey);
977         int i;
978 
979         for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
980                 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) &&
981                     (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) &&
982                     test_bit(i, lgr->rtokens_used_mask)) {
983                         /* already in list */
984                         return i;
985                 }
986         }
987         i = smc_rmb_reserve_rtoken_idx(lgr);
988         if (i < 0)
989                 return i;
990         lgr->rtokens[i][SMC_SINGLE_LINK].rkey = rkey;
991         lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = dma_addr;
992         return i;
993 }
994 
995 /* delete an rtoken */
996 int smc_rtoken_delete(struct smc_link_group *lgr, __be32 nw_rkey)
997 {
998         u32 rkey = ntohl(nw_rkey);
999         int i;
1000 
1001         for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
1002                 if (lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey &&
1003                     test_bit(i, lgr->rtokens_used_mask)) {
1004                         lgr->rtokens[i][SMC_SINGLE_LINK].rkey = 0;
1005                         lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = 0;
1006 
1007                         clear_bit(i, lgr->rtokens_used_mask);
1008                         return 0;
1009                 }
1010         }
1011         return -ENOENT;
1012 }
1013 
1014 /* save rkey and dma_addr received from peer during clc handshake */
1015 int smc_rmb_rtoken_handling(struct smc_connection *conn,
1016                             struct smc_clc_msg_accept_confirm *clc)
1017 {
1018         conn->rtoken_idx = smc_rtoken_add(conn->lgr, clc->rmb_dma_addr,
1019                                           clc->rmb_rkey);
1020         if (conn->rtoken_idx < 0)
1021                 return conn->rtoken_idx;
1022         return 0;
1023 }
1024 
1025 /* Called (from smc_exit) when module is removed */
1026 void smc_core_exit(void)
1027 {
1028         struct smc_link_group *lgr, *lg;
1029         LIST_HEAD(lgr_freeing_list);
1030 
1031         spin_lock_bh(&smc_lgr_list.lock);
1032         if (!list_empty(&smc_lgr_list.list))
1033                 list_splice_init(&smc_lgr_list.list, &lgr_freeing_list);
1034         spin_unlock_bh(&smc_lgr_list.lock);
1035         list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) {
1036                 list_del_init(&lgr->list);
1037                 if (!lgr->is_smcd) {
1038                         struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
1039 
1040                         if (lnk->state == SMC_LNK_ACTIVE)
1041                                 smc_llc_send_delete_link(lnk, SMC_LLC_REQ,
1042                                                          false);
1043                         smc_llc_link_inactive(lnk);
1044                 }
1045                 cancel_delayed_work_sync(&lgr->free_work);
1046                 if (lgr->is_smcd)
1047                         smc_ism_signal_shutdown(lgr);
1048                 smc_lgr_free(lgr); /* free link group */
1049         }
1050 }
1051 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp