~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/smc/smc_wr.c

Version: ~ [ linux-5.5 ] ~ [ linux-5.4.15 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.98 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.167 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.211 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.211 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Shared Memory Communications over RDMA (SMC-R) and RoCE
  4  *
  5  * Work Requests exploiting Infiniband API
  6  *
  7  * Work requests (WR) of type ib_post_send or ib_post_recv respectively
  8  * are submitted to either RC SQ or RC RQ respectively
  9  * (reliably connected send/receive queue)
 10  * and become work queue entries (WQEs).
 11  * While an SQ WR/WQE is pending, we track it until transmission completion.
 12  * Through a send or receive completion queue (CQ) respectively,
 13  * we get completion queue entries (CQEs) [aka work completions (WCs)].
 14  * Since the CQ callback is called from IRQ context, we split work by using
 15  * bottom halves implemented by tasklets.
 16  *
 17  * SMC uses this to exchange LLC (link layer control)
 18  * and CDC (connection data control) messages.
 19  *
 20  * Copyright IBM Corp. 2016
 21  *
 22  * Author(s):  Steffen Maier <maier@linux.vnet.ibm.com>
 23  */
 24 
 25 #include <linux/atomic.h>
 26 #include <linux/hashtable.h>
 27 #include <linux/wait.h>
 28 #include <rdma/ib_verbs.h>
 29 #include <asm/div64.h>
 30 
 31 #include "smc.h"
 32 #include "smc_wr.h"
 33 
 34 #define SMC_WR_MAX_POLL_CQE 10  /* max. # of compl. queue elements in 1 poll */
 35 
 36 #define SMC_WR_RX_HASH_BITS 4
 37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
 38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
 39 
 40 struct smc_wr_tx_pend { /* control data for a pending send request */
 41         u64                     wr_id;          /* work request id sent */
 42         smc_wr_tx_handler       handler;
 43         enum ib_wc_status       wc_status;      /* CQE status */
 44         struct smc_link         *link;
 45         u32                     idx;
 46         struct smc_wr_tx_pend_priv priv;
 47 };
 48 
 49 /******************************** send queue *********************************/
 50 
 51 /*------------------------------- completion --------------------------------*/
 52 
 53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
 54 {
 55         u32 i;
 56 
 57         for (i = 0; i < link->wr_tx_cnt; i++) {
 58                 if (link->wr_tx_pends[i].wr_id == wr_id)
 59                         return i;
 60         }
 61         return link->wr_tx_cnt;
 62 }
 63 
 64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
 65 {
 66         struct smc_wr_tx_pend pnd_snd;
 67         struct smc_link *link;
 68         u32 pnd_snd_idx;
 69         int i;
 70 
 71         link = wc->qp->qp_context;
 72 
 73         if (wc->opcode == IB_WC_REG_MR) {
 74                 if (wc->status)
 75                         link->wr_reg_state = FAILED;
 76                 else
 77                         link->wr_reg_state = CONFIRMED;
 78                 wake_up(&link->wr_reg_wait);
 79                 return;
 80         }
 81 
 82         pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
 83         if (pnd_snd_idx == link->wr_tx_cnt)
 84                 return;
 85         link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
 86         memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
 87         /* clear the full struct smc_wr_tx_pend including .priv */
 88         memset(&link->wr_tx_pends[pnd_snd_idx], 0,
 89                sizeof(link->wr_tx_pends[pnd_snd_idx]));
 90         memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
 91                sizeof(link->wr_tx_bufs[pnd_snd_idx]));
 92         if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
 93                 return;
 94         if (wc->status) {
 95                 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
 96                         /* clear full struct smc_wr_tx_pend including .priv */
 97                         memset(&link->wr_tx_pends[i], 0,
 98                                sizeof(link->wr_tx_pends[i]));
 99                         memset(&link->wr_tx_bufs[i], 0,
100                                sizeof(link->wr_tx_bufs[i]));
101                         clear_bit(i, link->wr_tx_mask);
102                 }
103                 /* terminate connections of this link group abnormally */
104                 smc_lgr_terminate(smc_get_lgr(link));
105         }
106         if (pnd_snd.handler)
107                 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
108         wake_up(&link->wr_tx_wait);
109 }
110 
111 static void smc_wr_tx_tasklet_fn(unsigned long data)
112 {
113         struct smc_ib_device *dev = (struct smc_ib_device *)data;
114         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
115         int i = 0, rc;
116         int polled = 0;
117 
118 again:
119         polled++;
120         do {
121                 memset(&wc, 0, sizeof(wc));
122                 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
123                 if (polled == 1) {
124                         ib_req_notify_cq(dev->roce_cq_send,
125                                          IB_CQ_NEXT_COMP |
126                                          IB_CQ_REPORT_MISSED_EVENTS);
127                 }
128                 if (!rc)
129                         break;
130                 for (i = 0; i < rc; i++)
131                         smc_wr_tx_process_cqe(&wc[i]);
132         } while (rc > 0);
133         if (polled == 1)
134                 goto again;
135 }
136 
137 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
138 {
139         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
140 
141         tasklet_schedule(&dev->send_tasklet);
142 }
143 
144 /*---------------------------- request submission ---------------------------*/
145 
146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
147 {
148         *idx = link->wr_tx_cnt;
149         for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
150                 if (!test_and_set_bit(*idx, link->wr_tx_mask))
151                         return 0;
152         }
153         *idx = link->wr_tx_cnt;
154         return -EBUSY;
155 }
156 
157 /**
158  * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
159  *                      and sets info for pending transmit tracking
160  * @link:               Pointer to smc_link used to later send the message.
161  * @handler:            Send completion handler function pointer.
162  * @wr_buf:             Out value returns pointer to message buffer.
163  * @wr_pend_priv:       Out value returns pointer serving as handler context.
164  *
165  * Return: 0 on success, or -errno on error.
166  */
167 int smc_wr_tx_get_free_slot(struct smc_link *link,
168                             smc_wr_tx_handler handler,
169                             struct smc_wr_buf **wr_buf,
170                             struct smc_wr_tx_pend_priv **wr_pend_priv)
171 {
172         struct smc_wr_tx_pend *wr_pend;
173         u32 idx = link->wr_tx_cnt;
174         struct ib_send_wr *wr_ib;
175         u64 wr_id;
176         int rc;
177 
178         *wr_buf = NULL;
179         *wr_pend_priv = NULL;
180         if (in_softirq()) {
181                 rc = smc_wr_tx_get_free_slot_index(link, &idx);
182                 if (rc)
183                         return rc;
184         } else {
185                 rc = wait_event_timeout(
186                         link->wr_tx_wait,
187                         link->state == SMC_LNK_INACTIVE ||
188                         (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
189                         SMC_WR_TX_WAIT_FREE_SLOT_TIME);
190                 if (!rc) {
191                         /* timeout - terminate connections */
192                         smc_lgr_terminate(smc_get_lgr(link));
193                         return -EPIPE;
194                 }
195                 if (idx == link->wr_tx_cnt)
196                         return -EPIPE;
197         }
198         wr_id = smc_wr_tx_get_next_wr_id(link);
199         wr_pend = &link->wr_tx_pends[idx];
200         wr_pend->wr_id = wr_id;
201         wr_pend->handler = handler;
202         wr_pend->link = link;
203         wr_pend->idx = idx;
204         wr_ib = &link->wr_tx_ibs[idx];
205         wr_ib->wr_id = wr_id;
206         *wr_buf = &link->wr_tx_bufs[idx];
207         *wr_pend_priv = &wr_pend->priv;
208         return 0;
209 }
210 
211 int smc_wr_tx_put_slot(struct smc_link *link,
212                        struct smc_wr_tx_pend_priv *wr_pend_priv)
213 {
214         struct smc_wr_tx_pend *pend;
215 
216         pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
217         if (pend->idx < link->wr_tx_cnt) {
218                 u32 idx = pend->idx;
219 
220                 /* clear the full struct smc_wr_tx_pend including .priv */
221                 memset(&link->wr_tx_pends[pend->idx], 0,
222                        sizeof(link->wr_tx_pends[pend->idx]));
223                 memset(&link->wr_tx_bufs[pend->idx], 0,
224                        sizeof(link->wr_tx_bufs[pend->idx]));
225                 test_and_clear_bit(idx, link->wr_tx_mask);
226                 return 1;
227         }
228 
229         return 0;
230 }
231 
232 /* Send prepared WR slot via ib_post_send.
233  * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
234  */
235 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
236 {
237         struct smc_wr_tx_pend *pend;
238         int rc;
239 
240         ib_req_notify_cq(link->smcibdev->roce_cq_send,
241                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
242         pend = container_of(priv, struct smc_wr_tx_pend, priv);
243         rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
244         if (rc) {
245                 smc_wr_tx_put_slot(link, priv);
246                 smc_lgr_terminate(smc_get_lgr(link));
247         }
248         return rc;
249 }
250 
251 /* Register a memory region and wait for result. */
252 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
253 {
254         int rc;
255 
256         ib_req_notify_cq(link->smcibdev->roce_cq_send,
257                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
258         link->wr_reg_state = POSTED;
259         link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
260         link->wr_reg.mr = mr;
261         link->wr_reg.key = mr->rkey;
262         rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
263         if (rc)
264                 return rc;
265 
266         rc = wait_event_interruptible_timeout(link->wr_reg_wait,
267                                               (link->wr_reg_state != POSTED),
268                                               SMC_WR_REG_MR_WAIT_TIME);
269         if (!rc) {
270                 /* timeout - terminate connections */
271                 smc_lgr_terminate(smc_get_lgr(link));
272                 return -EPIPE;
273         }
274         if (rc == -ERESTARTSYS)
275                 return -EINTR;
276         switch (link->wr_reg_state) {
277         case CONFIRMED:
278                 rc = 0;
279                 break;
280         case FAILED:
281                 rc = -EIO;
282                 break;
283         case POSTED:
284                 rc = -EPIPE;
285                 break;
286         }
287         return rc;
288 }
289 
290 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
291                              smc_wr_tx_filter filter,
292                              smc_wr_tx_dismisser dismisser,
293                              unsigned long data)
294 {
295         struct smc_wr_tx_pend_priv *tx_pend;
296         struct smc_wr_rx_hdr *wr_tx;
297         int i;
298 
299         for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
300                 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
301                 if (wr_tx->type != wr_tx_hdr_type)
302                         continue;
303                 tx_pend = &link->wr_tx_pends[i].priv;
304                 if (filter(tx_pend, data))
305                         dismisser(tx_pend);
306         }
307 }
308 
309 /****************************** receive queue ********************************/
310 
311 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
312 {
313         struct smc_wr_rx_handler *h_iter;
314         int rc = 0;
315 
316         spin_lock(&smc_wr_rx_hash_lock);
317         hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
318                 if (h_iter->type == handler->type) {
319                         rc = -EEXIST;
320                         goto out_unlock;
321                 }
322         }
323         hash_add(smc_wr_rx_hash, &handler->list, handler->type);
324 out_unlock:
325         spin_unlock(&smc_wr_rx_hash_lock);
326         return rc;
327 }
328 
329 /* Demultiplex a received work request based on the message type to its handler.
330  * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
331  * and not being modified any more afterwards so we don't need to lock it.
332  */
333 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
334 {
335         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
336         struct smc_wr_rx_handler *handler;
337         struct smc_wr_rx_hdr *wr_rx;
338         u64 temp_wr_id;
339         u32 index;
340 
341         if (wc->byte_len < sizeof(*wr_rx))
342                 return; /* short message */
343         temp_wr_id = wc->wr_id;
344         index = do_div(temp_wr_id, link->wr_rx_cnt);
345         wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
346         hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
347                 if (handler->type == wr_rx->type)
348                         handler->handler(wc, wr_rx);
349         }
350 }
351 
352 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
353 {
354         struct smc_link *link;
355         int i;
356 
357         for (i = 0; i < num; i++) {
358                 link = wc[i].qp->qp_context;
359                 if (wc[i].status == IB_WC_SUCCESS) {
360                         link->wr_rx_tstamp = jiffies;
361                         smc_wr_rx_demultiplex(&wc[i]);
362                         smc_wr_rx_post(link); /* refill WR RX */
363                 } else {
364                         /* handle status errors */
365                         switch (wc[i].status) {
366                         case IB_WC_RETRY_EXC_ERR:
367                         case IB_WC_RNR_RETRY_EXC_ERR:
368                         case IB_WC_WR_FLUSH_ERR:
369                                 /* terminate connections of this link group
370                                  * abnormally
371                                  */
372                                 smc_lgr_terminate(smc_get_lgr(link));
373                                 break;
374                         default:
375                                 smc_wr_rx_post(link); /* refill WR RX */
376                                 break;
377                         }
378                 }
379         }
380 }
381 
382 static void smc_wr_rx_tasklet_fn(unsigned long data)
383 {
384         struct smc_ib_device *dev = (struct smc_ib_device *)data;
385         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
386         int polled = 0;
387         int rc;
388 
389 again:
390         polled++;
391         do {
392                 memset(&wc, 0, sizeof(wc));
393                 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
394                 if (polled == 1) {
395                         ib_req_notify_cq(dev->roce_cq_recv,
396                                          IB_CQ_SOLICITED_MASK
397                                          | IB_CQ_REPORT_MISSED_EVENTS);
398                 }
399                 if (!rc)
400                         break;
401                 smc_wr_rx_process_cqes(&wc[0], rc);
402         } while (rc > 0);
403         if (polled == 1)
404                 goto again;
405 }
406 
407 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
408 {
409         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
410 
411         tasklet_schedule(&dev->recv_tasklet);
412 }
413 
414 int smc_wr_rx_post_init(struct smc_link *link)
415 {
416         u32 i;
417         int rc = 0;
418 
419         for (i = 0; i < link->wr_rx_cnt; i++)
420                 rc = smc_wr_rx_post(link);
421         return rc;
422 }
423 
424 /***************************** init, exit, misc ******************************/
425 
426 void smc_wr_remember_qp_attr(struct smc_link *lnk)
427 {
428         struct ib_qp_attr *attr = &lnk->qp_attr;
429         struct ib_qp_init_attr init_attr;
430 
431         memset(attr, 0, sizeof(*attr));
432         memset(&init_attr, 0, sizeof(init_attr));
433         ib_query_qp(lnk->roce_qp, attr,
434                     IB_QP_STATE |
435                     IB_QP_CUR_STATE |
436                     IB_QP_PKEY_INDEX |
437                     IB_QP_PORT |
438                     IB_QP_QKEY |
439                     IB_QP_AV |
440                     IB_QP_PATH_MTU |
441                     IB_QP_TIMEOUT |
442                     IB_QP_RETRY_CNT |
443                     IB_QP_RNR_RETRY |
444                     IB_QP_RQ_PSN |
445                     IB_QP_ALT_PATH |
446                     IB_QP_MIN_RNR_TIMER |
447                     IB_QP_SQ_PSN |
448                     IB_QP_PATH_MIG_STATE |
449                     IB_QP_CAP |
450                     IB_QP_DEST_QPN,
451                     &init_attr);
452 
453         lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
454                                lnk->qp_attr.cap.max_send_wr);
455         lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
456                                lnk->qp_attr.cap.max_recv_wr);
457 }
458 
459 static void smc_wr_init_sge(struct smc_link *lnk)
460 {
461         u32 i;
462 
463         for (i = 0; i < lnk->wr_tx_cnt; i++) {
464                 lnk->wr_tx_sges[i].addr =
465                         lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
466                 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
467                 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
468                 lnk->wr_tx_ibs[i].next = NULL;
469                 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
470                 lnk->wr_tx_ibs[i].num_sge = 1;
471                 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
472                 lnk->wr_tx_ibs[i].send_flags =
473                         IB_SEND_SIGNALED | IB_SEND_SOLICITED;
474         }
475         for (i = 0; i < lnk->wr_rx_cnt; i++) {
476                 lnk->wr_rx_sges[i].addr =
477                         lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
478                 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
479                 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
480                 lnk->wr_rx_ibs[i].next = NULL;
481                 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
482                 lnk->wr_rx_ibs[i].num_sge = 1;
483         }
484         lnk->wr_reg.wr.next = NULL;
485         lnk->wr_reg.wr.num_sge = 0;
486         lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
487         lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
488         lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
489 }
490 
491 void smc_wr_free_link(struct smc_link *lnk)
492 {
493         struct ib_device *ibdev;
494 
495         memset(lnk->wr_tx_mask, 0,
496                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
497 
498         if (!lnk->smcibdev)
499                 return;
500         ibdev = lnk->smcibdev->ibdev;
501 
502         if (lnk->wr_rx_dma_addr) {
503                 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
504                                     SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
505                                     DMA_FROM_DEVICE);
506                 lnk->wr_rx_dma_addr = 0;
507         }
508         if (lnk->wr_tx_dma_addr) {
509                 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
510                                     SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
511                                     DMA_TO_DEVICE);
512                 lnk->wr_tx_dma_addr = 0;
513         }
514 }
515 
516 void smc_wr_free_link_mem(struct smc_link *lnk)
517 {
518         kfree(lnk->wr_tx_pends);
519         lnk->wr_tx_pends = NULL;
520         kfree(lnk->wr_tx_mask);
521         lnk->wr_tx_mask = NULL;
522         kfree(lnk->wr_tx_sges);
523         lnk->wr_tx_sges = NULL;
524         kfree(lnk->wr_rx_sges);
525         lnk->wr_rx_sges = NULL;
526         kfree(lnk->wr_rx_ibs);
527         lnk->wr_rx_ibs = NULL;
528         kfree(lnk->wr_tx_ibs);
529         lnk->wr_tx_ibs = NULL;
530         kfree(lnk->wr_tx_bufs);
531         lnk->wr_tx_bufs = NULL;
532         kfree(lnk->wr_rx_bufs);
533         lnk->wr_rx_bufs = NULL;
534 }
535 
536 int smc_wr_alloc_link_mem(struct smc_link *link)
537 {
538         /* allocate link related memory */
539         link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
540         if (!link->wr_tx_bufs)
541                 goto no_mem;
542         link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
543                                    GFP_KERNEL);
544         if (!link->wr_rx_bufs)
545                 goto no_mem_wr_tx_bufs;
546         link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
547                                   GFP_KERNEL);
548         if (!link->wr_tx_ibs)
549                 goto no_mem_wr_rx_bufs;
550         link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
551                                   sizeof(link->wr_rx_ibs[0]),
552                                   GFP_KERNEL);
553         if (!link->wr_rx_ibs)
554                 goto no_mem_wr_tx_ibs;
555         link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
556                                    GFP_KERNEL);
557         if (!link->wr_tx_sges)
558                 goto no_mem_wr_rx_ibs;
559         link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
560                                    sizeof(link->wr_rx_sges[0]),
561                                    GFP_KERNEL);
562         if (!link->wr_rx_sges)
563                 goto no_mem_wr_tx_sges;
564         link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
565                                    sizeof(*link->wr_tx_mask),
566                                    GFP_KERNEL);
567         if (!link->wr_tx_mask)
568                 goto no_mem_wr_rx_sges;
569         link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
570                                     sizeof(link->wr_tx_pends[0]),
571                                     GFP_KERNEL);
572         if (!link->wr_tx_pends)
573                 goto no_mem_wr_tx_mask;
574         return 0;
575 
576 no_mem_wr_tx_mask:
577         kfree(link->wr_tx_mask);
578 no_mem_wr_rx_sges:
579         kfree(link->wr_rx_sges);
580 no_mem_wr_tx_sges:
581         kfree(link->wr_tx_sges);
582 no_mem_wr_rx_ibs:
583         kfree(link->wr_rx_ibs);
584 no_mem_wr_tx_ibs:
585         kfree(link->wr_tx_ibs);
586 no_mem_wr_rx_bufs:
587         kfree(link->wr_rx_bufs);
588 no_mem_wr_tx_bufs:
589         kfree(link->wr_tx_bufs);
590 no_mem:
591         return -ENOMEM;
592 }
593 
594 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
595 {
596         tasklet_kill(&smcibdev->recv_tasklet);
597         tasklet_kill(&smcibdev->send_tasklet);
598 }
599 
600 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
601 {
602         tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
603                      (unsigned long)smcibdev);
604         tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
605                      (unsigned long)smcibdev);
606 }
607 
608 int smc_wr_create_link(struct smc_link *lnk)
609 {
610         struct ib_device *ibdev = lnk->smcibdev->ibdev;
611         int rc = 0;
612 
613         smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
614         lnk->wr_rx_id = 0;
615         lnk->wr_rx_dma_addr = ib_dma_map_single(
616                 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
617                 DMA_FROM_DEVICE);
618         if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
619                 lnk->wr_rx_dma_addr = 0;
620                 rc = -EIO;
621                 goto out;
622         }
623         lnk->wr_tx_dma_addr = ib_dma_map_single(
624                 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
625                 DMA_TO_DEVICE);
626         if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
627                 rc = -EIO;
628                 goto dma_unmap;
629         }
630         smc_wr_init_sge(lnk);
631         memset(lnk->wr_tx_mask, 0,
632                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
633         init_waitqueue_head(&lnk->wr_tx_wait);
634         init_waitqueue_head(&lnk->wr_reg_wait);
635         return rc;
636 
637 dma_unmap:
638         ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
639                             SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
640                             DMA_FROM_DEVICE);
641         lnk->wr_rx_dma_addr = 0;
642 out:
643         return rc;
644 }
645 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp