~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/smc/smc_wr.c

Version: ~ [ linux-5.6-rc1 ] ~ [ linux-5.5.2 ] ~ [ linux-5.4.17 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.102 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.170 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.213 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.213 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Shared Memory Communications over RDMA (SMC-R) and RoCE
  4  *
  5  * Work Requests exploiting Infiniband API
  6  *
  7  * Work requests (WR) of type ib_post_send or ib_post_recv respectively
  8  * are submitted to either RC SQ or RC RQ respectively
  9  * (reliably connected send/receive queue)
 10  * and become work queue entries (WQEs).
 11  * While an SQ WR/WQE is pending, we track it until transmission completion.
 12  * Through a send or receive completion queue (CQ) respectively,
 13  * we get completion queue entries (CQEs) [aka work completions (WCs)].
 14  * Since the CQ callback is called from IRQ context, we split work by using
 15  * bottom halves implemented by tasklets.
 16  *
 17  * SMC uses this to exchange LLC (link layer control)
 18  * and CDC (connection data control) messages.
 19  *
 20  * Copyright IBM Corp. 2016
 21  *
 22  * Author(s):  Steffen Maier <maier@linux.vnet.ibm.com>
 23  */
 24 
 25 #include <linux/atomic.h>
 26 #include <linux/hashtable.h>
 27 #include <linux/wait.h>
 28 #include <rdma/ib_verbs.h>
 29 #include <asm/div64.h>
 30 
 31 #include "smc.h"
 32 #include "smc_wr.h"
 33 
 34 #define SMC_WR_MAX_POLL_CQE 10  /* max. # of compl. queue elements in 1 poll */
 35 
 36 #define SMC_WR_RX_HASH_BITS 4
 37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
 38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
 39 
 40 struct smc_wr_tx_pend { /* control data for a pending send request */
 41         u64                     wr_id;          /* work request id sent */
 42         smc_wr_tx_handler       handler;
 43         enum ib_wc_status       wc_status;      /* CQE status */
 44         struct smc_link         *link;
 45         u32                     idx;
 46         struct smc_wr_tx_pend_priv priv;
 47 };
 48 
 49 /******************************** send queue *********************************/
 50 
 51 /*------------------------------- completion --------------------------------*/
 52 
 53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
 54 {
 55         u32 i;
 56 
 57         for (i = 0; i < link->wr_tx_cnt; i++) {
 58                 if (link->wr_tx_pends[i].wr_id == wr_id)
 59                         return i;
 60         }
 61         return link->wr_tx_cnt;
 62 }
 63 
 64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
 65 {
 66         struct smc_wr_tx_pend pnd_snd;
 67         struct smc_link *link;
 68         u32 pnd_snd_idx;
 69         int i;
 70 
 71         link = wc->qp->qp_context;
 72 
 73         if (wc->opcode == IB_WC_REG_MR) {
 74                 if (wc->status)
 75                         link->wr_reg_state = FAILED;
 76                 else
 77                         link->wr_reg_state = CONFIRMED;
 78                 wake_up(&link->wr_reg_wait);
 79                 return;
 80         }
 81 
 82         pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
 83         if (pnd_snd_idx == link->wr_tx_cnt)
 84                 return;
 85         link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
 86         memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
 87         /* clear the full struct smc_wr_tx_pend including .priv */
 88         memset(&link->wr_tx_pends[pnd_snd_idx], 0,
 89                sizeof(link->wr_tx_pends[pnd_snd_idx]));
 90         memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
 91                sizeof(link->wr_tx_bufs[pnd_snd_idx]));
 92         if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
 93                 return;
 94         if (wc->status) {
 95                 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
 96                         /* clear full struct smc_wr_tx_pend including .priv */
 97                         memset(&link->wr_tx_pends[i], 0,
 98                                sizeof(link->wr_tx_pends[i]));
 99                         memset(&link->wr_tx_bufs[i], 0,
100                                sizeof(link->wr_tx_bufs[i]));
101                         clear_bit(i, link->wr_tx_mask);
102                 }
103                 /* terminate connections of this link group abnormally */
104                 smc_lgr_terminate(smc_get_lgr(link));
105         }
106         if (pnd_snd.handler)
107                 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
108         wake_up(&link->wr_tx_wait);
109 }
110 
111 static void smc_wr_tx_tasklet_fn(unsigned long data)
112 {
113         struct smc_ib_device *dev = (struct smc_ib_device *)data;
114         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
115         int i = 0, rc;
116         int polled = 0;
117 
118 again:
119         polled++;
120         do {
121                 memset(&wc, 0, sizeof(wc));
122                 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
123                 if (polled == 1) {
124                         ib_req_notify_cq(dev->roce_cq_send,
125                                          IB_CQ_NEXT_COMP |
126                                          IB_CQ_REPORT_MISSED_EVENTS);
127                 }
128                 if (!rc)
129                         break;
130                 for (i = 0; i < rc; i++)
131                         smc_wr_tx_process_cqe(&wc[i]);
132         } while (rc > 0);
133         if (polled == 1)
134                 goto again;
135 }
136 
137 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
138 {
139         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
140 
141         tasklet_schedule(&dev->send_tasklet);
142 }
143 
144 /*---------------------------- request submission ---------------------------*/
145 
146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
147 {
148         *idx = link->wr_tx_cnt;
149         for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
150                 if (!test_and_set_bit(*idx, link->wr_tx_mask))
151                         return 0;
152         }
153         *idx = link->wr_tx_cnt;
154         return -EBUSY;
155 }
156 
157 /**
158  * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
159  *                      and sets info for pending transmit tracking
160  * @link:               Pointer to smc_link used to later send the message.
161  * @handler:            Send completion handler function pointer.
162  * @wr_buf:             Out value returns pointer to message buffer.
163  * @wr_rdma_buf:        Out value returns pointer to rdma work request.
164  * @wr_pend_priv:       Out value returns pointer serving as handler context.
165  *
166  * Return: 0 on success, or -errno on error.
167  */
168 int smc_wr_tx_get_free_slot(struct smc_link *link,
169                             smc_wr_tx_handler handler,
170                             struct smc_wr_buf **wr_buf,
171                             struct smc_rdma_wr **wr_rdma_buf,
172                             struct smc_wr_tx_pend_priv **wr_pend_priv)
173 {
174         struct smc_wr_tx_pend *wr_pend;
175         u32 idx = link->wr_tx_cnt;
176         struct ib_send_wr *wr_ib;
177         u64 wr_id;
178         int rc;
179 
180         *wr_buf = NULL;
181         *wr_pend_priv = NULL;
182         if (in_softirq()) {
183                 rc = smc_wr_tx_get_free_slot_index(link, &idx);
184                 if (rc)
185                         return rc;
186         } else {
187                 rc = wait_event_timeout(
188                         link->wr_tx_wait,
189                         link->state == SMC_LNK_INACTIVE ||
190                         (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
191                         SMC_WR_TX_WAIT_FREE_SLOT_TIME);
192                 if (!rc) {
193                         /* timeout - terminate connections */
194                         smc_lgr_terminate(smc_get_lgr(link));
195                         return -EPIPE;
196                 }
197                 if (idx == link->wr_tx_cnt)
198                         return -EPIPE;
199         }
200         wr_id = smc_wr_tx_get_next_wr_id(link);
201         wr_pend = &link->wr_tx_pends[idx];
202         wr_pend->wr_id = wr_id;
203         wr_pend->handler = handler;
204         wr_pend->link = link;
205         wr_pend->idx = idx;
206         wr_ib = &link->wr_tx_ibs[idx];
207         wr_ib->wr_id = wr_id;
208         *wr_buf = &link->wr_tx_bufs[idx];
209         if (wr_rdma_buf)
210                 *wr_rdma_buf = &link->wr_tx_rdmas[idx];
211         *wr_pend_priv = &wr_pend->priv;
212         return 0;
213 }
214 
215 int smc_wr_tx_put_slot(struct smc_link *link,
216                        struct smc_wr_tx_pend_priv *wr_pend_priv)
217 {
218         struct smc_wr_tx_pend *pend;
219 
220         pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
221         if (pend->idx < link->wr_tx_cnt) {
222                 u32 idx = pend->idx;
223 
224                 /* clear the full struct smc_wr_tx_pend including .priv */
225                 memset(&link->wr_tx_pends[idx], 0,
226                        sizeof(link->wr_tx_pends[idx]));
227                 memset(&link->wr_tx_bufs[idx], 0,
228                        sizeof(link->wr_tx_bufs[idx]));
229                 test_and_clear_bit(idx, link->wr_tx_mask);
230                 return 1;
231         }
232 
233         return 0;
234 }
235 
236 /* Send prepared WR slot via ib_post_send.
237  * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
238  */
239 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
240 {
241         struct smc_wr_tx_pend *pend;
242         int rc;
243 
244         ib_req_notify_cq(link->smcibdev->roce_cq_send,
245                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
246         pend = container_of(priv, struct smc_wr_tx_pend, priv);
247         rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
248         if (rc) {
249                 smc_wr_tx_put_slot(link, priv);
250                 smc_lgr_terminate(smc_get_lgr(link));
251         }
252         return rc;
253 }
254 
255 /* Register a memory region and wait for result. */
256 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
257 {
258         int rc;
259 
260         ib_req_notify_cq(link->smcibdev->roce_cq_send,
261                          IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
262         link->wr_reg_state = POSTED;
263         link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
264         link->wr_reg.mr = mr;
265         link->wr_reg.key = mr->rkey;
266         rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
267         if (rc)
268                 return rc;
269 
270         rc = wait_event_interruptible_timeout(link->wr_reg_wait,
271                                               (link->wr_reg_state != POSTED),
272                                               SMC_WR_REG_MR_WAIT_TIME);
273         if (!rc) {
274                 /* timeout - terminate connections */
275                 smc_lgr_terminate(smc_get_lgr(link));
276                 return -EPIPE;
277         }
278         if (rc == -ERESTARTSYS)
279                 return -EINTR;
280         switch (link->wr_reg_state) {
281         case CONFIRMED:
282                 rc = 0;
283                 break;
284         case FAILED:
285                 rc = -EIO;
286                 break;
287         case POSTED:
288                 rc = -EPIPE;
289                 break;
290         }
291         return rc;
292 }
293 
294 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
295                              smc_wr_tx_filter filter,
296                              smc_wr_tx_dismisser dismisser,
297                              unsigned long data)
298 {
299         struct smc_wr_tx_pend_priv *tx_pend;
300         struct smc_wr_rx_hdr *wr_tx;
301         int i;
302 
303         for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
304                 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
305                 if (wr_tx->type != wr_tx_hdr_type)
306                         continue;
307                 tx_pend = &link->wr_tx_pends[i].priv;
308                 if (filter(tx_pend, data))
309                         dismisser(tx_pend);
310         }
311 }
312 
313 /****************************** receive queue ********************************/
314 
315 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
316 {
317         struct smc_wr_rx_handler *h_iter;
318         int rc = 0;
319 
320         spin_lock(&smc_wr_rx_hash_lock);
321         hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
322                 if (h_iter->type == handler->type) {
323                         rc = -EEXIST;
324                         goto out_unlock;
325                 }
326         }
327         hash_add(smc_wr_rx_hash, &handler->list, handler->type);
328 out_unlock:
329         spin_unlock(&smc_wr_rx_hash_lock);
330         return rc;
331 }
332 
333 /* Demultiplex a received work request based on the message type to its handler.
334  * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
335  * and not being modified any more afterwards so we don't need to lock it.
336  */
337 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
338 {
339         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
340         struct smc_wr_rx_handler *handler;
341         struct smc_wr_rx_hdr *wr_rx;
342         u64 temp_wr_id;
343         u32 index;
344 
345         if (wc->byte_len < sizeof(*wr_rx))
346                 return; /* short message */
347         temp_wr_id = wc->wr_id;
348         index = do_div(temp_wr_id, link->wr_rx_cnt);
349         wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
350         hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
351                 if (handler->type == wr_rx->type)
352                         handler->handler(wc, wr_rx);
353         }
354 }
355 
356 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
357 {
358         struct smc_link *link;
359         int i;
360 
361         for (i = 0; i < num; i++) {
362                 link = wc[i].qp->qp_context;
363                 if (wc[i].status == IB_WC_SUCCESS) {
364                         link->wr_rx_tstamp = jiffies;
365                         smc_wr_rx_demultiplex(&wc[i]);
366                         smc_wr_rx_post(link); /* refill WR RX */
367                 } else {
368                         /* handle status errors */
369                         switch (wc[i].status) {
370                         case IB_WC_RETRY_EXC_ERR:
371                         case IB_WC_RNR_RETRY_EXC_ERR:
372                         case IB_WC_WR_FLUSH_ERR:
373                                 /* terminate connections of this link group
374                                  * abnormally
375                                  */
376                                 smc_lgr_terminate(smc_get_lgr(link));
377                                 break;
378                         default:
379                                 smc_wr_rx_post(link); /* refill WR RX */
380                                 break;
381                         }
382                 }
383         }
384 }
385 
386 static void smc_wr_rx_tasklet_fn(unsigned long data)
387 {
388         struct smc_ib_device *dev = (struct smc_ib_device *)data;
389         struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
390         int polled = 0;
391         int rc;
392 
393 again:
394         polled++;
395         do {
396                 memset(&wc, 0, sizeof(wc));
397                 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
398                 if (polled == 1) {
399                         ib_req_notify_cq(dev->roce_cq_recv,
400                                          IB_CQ_SOLICITED_MASK
401                                          | IB_CQ_REPORT_MISSED_EVENTS);
402                 }
403                 if (!rc)
404                         break;
405                 smc_wr_rx_process_cqes(&wc[0], rc);
406         } while (rc > 0);
407         if (polled == 1)
408                 goto again;
409 }
410 
411 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
412 {
413         struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
414 
415         tasklet_schedule(&dev->recv_tasklet);
416 }
417 
418 int smc_wr_rx_post_init(struct smc_link *link)
419 {
420         u32 i;
421         int rc = 0;
422 
423         for (i = 0; i < link->wr_rx_cnt; i++)
424                 rc = smc_wr_rx_post(link);
425         return rc;
426 }
427 
428 /***************************** init, exit, misc ******************************/
429 
430 void smc_wr_remember_qp_attr(struct smc_link *lnk)
431 {
432         struct ib_qp_attr *attr = &lnk->qp_attr;
433         struct ib_qp_init_attr init_attr;
434 
435         memset(attr, 0, sizeof(*attr));
436         memset(&init_attr, 0, sizeof(init_attr));
437         ib_query_qp(lnk->roce_qp, attr,
438                     IB_QP_STATE |
439                     IB_QP_CUR_STATE |
440                     IB_QP_PKEY_INDEX |
441                     IB_QP_PORT |
442                     IB_QP_QKEY |
443                     IB_QP_AV |
444                     IB_QP_PATH_MTU |
445                     IB_QP_TIMEOUT |
446                     IB_QP_RETRY_CNT |
447                     IB_QP_RNR_RETRY |
448                     IB_QP_RQ_PSN |
449                     IB_QP_ALT_PATH |
450                     IB_QP_MIN_RNR_TIMER |
451                     IB_QP_SQ_PSN |
452                     IB_QP_PATH_MIG_STATE |
453                     IB_QP_CAP |
454                     IB_QP_DEST_QPN,
455                     &init_attr);
456 
457         lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
458                                lnk->qp_attr.cap.max_send_wr);
459         lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
460                                lnk->qp_attr.cap.max_recv_wr);
461 }
462 
463 static void smc_wr_init_sge(struct smc_link *lnk)
464 {
465         u32 i;
466 
467         for (i = 0; i < lnk->wr_tx_cnt; i++) {
468                 lnk->wr_tx_sges[i].addr =
469                         lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
470                 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
471                 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
472                 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey =
473                         lnk->roce_pd->local_dma_lkey;
474                 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey =
475                         lnk->roce_pd->local_dma_lkey;
476                 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey =
477                         lnk->roce_pd->local_dma_lkey;
478                 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey =
479                         lnk->roce_pd->local_dma_lkey;
480                 lnk->wr_tx_ibs[i].next = NULL;
481                 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
482                 lnk->wr_tx_ibs[i].num_sge = 1;
483                 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
484                 lnk->wr_tx_ibs[i].send_flags =
485                         IB_SEND_SIGNALED | IB_SEND_SOLICITED;
486                 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE;
487                 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE;
488                 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list =
489                         lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge;
490                 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list =
491                         lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge;
492         }
493         for (i = 0; i < lnk->wr_rx_cnt; i++) {
494                 lnk->wr_rx_sges[i].addr =
495                         lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
496                 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
497                 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
498                 lnk->wr_rx_ibs[i].next = NULL;
499                 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
500                 lnk->wr_rx_ibs[i].num_sge = 1;
501         }
502         lnk->wr_reg.wr.next = NULL;
503         lnk->wr_reg.wr.num_sge = 0;
504         lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
505         lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
506         lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
507 }
508 
509 void smc_wr_free_link(struct smc_link *lnk)
510 {
511         struct ib_device *ibdev;
512 
513         memset(lnk->wr_tx_mask, 0,
514                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
515 
516         if (!lnk->smcibdev)
517                 return;
518         ibdev = lnk->smcibdev->ibdev;
519 
520         if (lnk->wr_rx_dma_addr) {
521                 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
522                                     SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
523                                     DMA_FROM_DEVICE);
524                 lnk->wr_rx_dma_addr = 0;
525         }
526         if (lnk->wr_tx_dma_addr) {
527                 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
528                                     SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
529                                     DMA_TO_DEVICE);
530                 lnk->wr_tx_dma_addr = 0;
531         }
532 }
533 
534 void smc_wr_free_link_mem(struct smc_link *lnk)
535 {
536         kfree(lnk->wr_tx_pends);
537         lnk->wr_tx_pends = NULL;
538         kfree(lnk->wr_tx_mask);
539         lnk->wr_tx_mask = NULL;
540         kfree(lnk->wr_tx_sges);
541         lnk->wr_tx_sges = NULL;
542         kfree(lnk->wr_tx_rdma_sges);
543         lnk->wr_tx_rdma_sges = NULL;
544         kfree(lnk->wr_rx_sges);
545         lnk->wr_rx_sges = NULL;
546         kfree(lnk->wr_tx_rdmas);
547         lnk->wr_tx_rdmas = NULL;
548         kfree(lnk->wr_rx_ibs);
549         lnk->wr_rx_ibs = NULL;
550         kfree(lnk->wr_tx_ibs);
551         lnk->wr_tx_ibs = NULL;
552         kfree(lnk->wr_tx_bufs);
553         lnk->wr_tx_bufs = NULL;
554         kfree(lnk->wr_rx_bufs);
555         lnk->wr_rx_bufs = NULL;
556 }
557 
558 int smc_wr_alloc_link_mem(struct smc_link *link)
559 {
560         /* allocate link related memory */
561         link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
562         if (!link->wr_tx_bufs)
563                 goto no_mem;
564         link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
565                                    GFP_KERNEL);
566         if (!link->wr_rx_bufs)
567                 goto no_mem_wr_tx_bufs;
568         link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
569                                   GFP_KERNEL);
570         if (!link->wr_tx_ibs)
571                 goto no_mem_wr_rx_bufs;
572         link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
573                                   sizeof(link->wr_rx_ibs[0]),
574                                   GFP_KERNEL);
575         if (!link->wr_rx_ibs)
576                 goto no_mem_wr_tx_ibs;
577         link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,
578                                     sizeof(link->wr_tx_rdmas[0]),
579                                     GFP_KERNEL);
580         if (!link->wr_tx_rdmas)
581                 goto no_mem_wr_rx_ibs;
582         link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,
583                                         sizeof(link->wr_tx_rdma_sges[0]),
584                                         GFP_KERNEL);
585         if (!link->wr_tx_rdma_sges)
586                 goto no_mem_wr_tx_rdmas;
587         link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
588                                    GFP_KERNEL);
589         if (!link->wr_tx_sges)
590                 goto no_mem_wr_tx_rdma_sges;
591         link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
592                                    sizeof(link->wr_rx_sges[0]),
593                                    GFP_KERNEL);
594         if (!link->wr_rx_sges)
595                 goto no_mem_wr_tx_sges;
596         link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
597                                    sizeof(*link->wr_tx_mask),
598                                    GFP_KERNEL);
599         if (!link->wr_tx_mask)
600                 goto no_mem_wr_rx_sges;
601         link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
602                                     sizeof(link->wr_tx_pends[0]),
603                                     GFP_KERNEL);
604         if (!link->wr_tx_pends)
605                 goto no_mem_wr_tx_mask;
606         return 0;
607 
608 no_mem_wr_tx_mask:
609         kfree(link->wr_tx_mask);
610 no_mem_wr_rx_sges:
611         kfree(link->wr_rx_sges);
612 no_mem_wr_tx_sges:
613         kfree(link->wr_tx_sges);
614 no_mem_wr_tx_rdma_sges:
615         kfree(link->wr_tx_rdma_sges);
616 no_mem_wr_tx_rdmas:
617         kfree(link->wr_tx_rdmas);
618 no_mem_wr_rx_ibs:
619         kfree(link->wr_rx_ibs);
620 no_mem_wr_tx_ibs:
621         kfree(link->wr_tx_ibs);
622 no_mem_wr_rx_bufs:
623         kfree(link->wr_rx_bufs);
624 no_mem_wr_tx_bufs:
625         kfree(link->wr_tx_bufs);
626 no_mem:
627         return -ENOMEM;
628 }
629 
630 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
631 {
632         tasklet_kill(&smcibdev->recv_tasklet);
633         tasklet_kill(&smcibdev->send_tasklet);
634 }
635 
636 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
637 {
638         tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
639                      (unsigned long)smcibdev);
640         tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
641                      (unsigned long)smcibdev);
642 }
643 
644 int smc_wr_create_link(struct smc_link *lnk)
645 {
646         struct ib_device *ibdev = lnk->smcibdev->ibdev;
647         int rc = 0;
648 
649         smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
650         lnk->wr_rx_id = 0;
651         lnk->wr_rx_dma_addr = ib_dma_map_single(
652                 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
653                 DMA_FROM_DEVICE);
654         if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
655                 lnk->wr_rx_dma_addr = 0;
656                 rc = -EIO;
657                 goto out;
658         }
659         lnk->wr_tx_dma_addr = ib_dma_map_single(
660                 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
661                 DMA_TO_DEVICE);
662         if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
663                 rc = -EIO;
664                 goto dma_unmap;
665         }
666         smc_wr_init_sge(lnk);
667         memset(lnk->wr_tx_mask, 0,
668                BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
669         init_waitqueue_head(&lnk->wr_tx_wait);
670         init_waitqueue_head(&lnk->wr_reg_wait);
671         return rc;
672 
673 dma_unmap:
674         ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
675                             SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
676                             DMA_FROM_DEVICE);
677         lnk->wr_rx_dma_addr = 0;
678 out:
679         return rc;
680 }
681 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp