~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/sunrpc/xprtrdma/transport.c

Version: ~ [ linux-5.12-rc1 ] ~ [ linux-5.11.2 ] ~ [ linux-5.10.19 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.101 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.177 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.222 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.258 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.258 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2 /*
  3  * Copyright (c) 2014-2017 Oracle.  All rights reserved.
  4  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
  5  *
  6  * This software is available to you under a choice of one of two
  7  * licenses.  You may choose to be licensed under the terms of the GNU
  8  * General Public License (GPL) Version 2, available from the file
  9  * COPYING in the main directory of this source tree, or the BSD-type
 10  * license below:
 11  *
 12  * Redistribution and use in source and binary forms, with or without
 13  * modification, are permitted provided that the following conditions
 14  * are met:
 15  *
 16  *      Redistributions of source code must retain the above copyright
 17  *      notice, this list of conditions and the following disclaimer.
 18  *
 19  *      Redistributions in binary form must reproduce the above
 20  *      copyright notice, this list of conditions and the following
 21  *      disclaimer in the documentation and/or other materials provided
 22  *      with the distribution.
 23  *
 24  *      Neither the name of the Network Appliance, Inc. nor the names of
 25  *      its contributors may be used to endorse or promote products
 26  *      derived from this software without specific prior written
 27  *      permission.
 28  *
 29  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 30  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 31  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 32  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 33  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 34  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 35  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 36  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 37  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 38  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 39  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 40  */
 41 
 42 /*
 43  * transport.c
 44  *
 45  * This file contains the top-level implementation of an RPC RDMA
 46  * transport.
 47  *
 48  * Naming convention: functions beginning with xprt_ are part of the
 49  * transport switch. All others are RPC RDMA internal.
 50  */
 51 
 52 #include <linux/module.h>
 53 #include <linux/slab.h>
 54 #include <linux/seq_file.h>
 55 #include <linux/smp.h>
 56 
 57 #include <linux/sunrpc/addr.h>
 58 #include <linux/sunrpc/svc_rdma.h>
 59 
 60 #include "xprt_rdma.h"
 61 #include <trace/events/rpcrdma.h>
 62 
 63 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 64 # define RPCDBG_FACILITY        RPCDBG_TRANS
 65 #endif
 66 
 67 /*
 68  * tunables
 69  */
 70 
 71 unsigned int xprt_rdma_slot_table_entries = RPCRDMA_DEF_SLOT_TABLE;
 72 unsigned int xprt_rdma_max_inline_read = RPCRDMA_DEF_INLINE;
 73 unsigned int xprt_rdma_max_inline_write = RPCRDMA_DEF_INLINE;
 74 unsigned int xprt_rdma_memreg_strategy          = RPCRDMA_FRWR;
 75 int xprt_rdma_pad_optimize;
 76 
 77 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 78 
 79 static unsigned int min_slot_table_size = RPCRDMA_MIN_SLOT_TABLE;
 80 static unsigned int max_slot_table_size = RPCRDMA_MAX_SLOT_TABLE;
 81 static unsigned int min_inline_size = RPCRDMA_MIN_INLINE;
 82 static unsigned int max_inline_size = RPCRDMA_MAX_INLINE;
 83 static unsigned int max_padding = PAGE_SIZE;
 84 static unsigned int min_memreg = RPCRDMA_BOUNCEBUFFERS;
 85 static unsigned int max_memreg = RPCRDMA_LAST - 1;
 86 static unsigned int dummy;
 87 
 88 static struct ctl_table_header *sunrpc_table_header;
 89 
 90 static struct ctl_table xr_tunables_table[] = {
 91         {
 92                 .procname       = "rdma_slot_table_entries",
 93                 .data           = &xprt_rdma_slot_table_entries,
 94                 .maxlen         = sizeof(unsigned int),
 95                 .mode           = 0644,
 96                 .proc_handler   = proc_dointvec_minmax,
 97                 .extra1         = &min_slot_table_size,
 98                 .extra2         = &max_slot_table_size
 99         },
100         {
101                 .procname       = "rdma_max_inline_read",
102                 .data           = &xprt_rdma_max_inline_read,
103                 .maxlen         = sizeof(unsigned int),
104                 .mode           = 0644,
105                 .proc_handler   = proc_dointvec_minmax,
106                 .extra1         = &min_inline_size,
107                 .extra2         = &max_inline_size,
108         },
109         {
110                 .procname       = "rdma_max_inline_write",
111                 .data           = &xprt_rdma_max_inline_write,
112                 .maxlen         = sizeof(unsigned int),
113                 .mode           = 0644,
114                 .proc_handler   = proc_dointvec_minmax,
115                 .extra1         = &min_inline_size,
116                 .extra2         = &max_inline_size,
117         },
118         {
119                 .procname       = "rdma_inline_write_padding",
120                 .data           = &dummy,
121                 .maxlen         = sizeof(unsigned int),
122                 .mode           = 0644,
123                 .proc_handler   = proc_dointvec_minmax,
124                 .extra1         = SYSCTL_ZERO,
125                 .extra2         = &max_padding,
126         },
127         {
128                 .procname       = "rdma_memreg_strategy",
129                 .data           = &xprt_rdma_memreg_strategy,
130                 .maxlen         = sizeof(unsigned int),
131                 .mode           = 0644,
132                 .proc_handler   = proc_dointvec_minmax,
133                 .extra1         = &min_memreg,
134                 .extra2         = &max_memreg,
135         },
136         {
137                 .procname       = "rdma_pad_optimize",
138                 .data           = &xprt_rdma_pad_optimize,
139                 .maxlen         = sizeof(unsigned int),
140                 .mode           = 0644,
141                 .proc_handler   = proc_dointvec,
142         },
143         { },
144 };
145 
146 static struct ctl_table sunrpc_table[] = {
147         {
148                 .procname       = "sunrpc",
149                 .mode           = 0555,
150                 .child          = xr_tunables_table
151         },
152         { },
153 };
154 
155 #endif
156 
157 static const struct rpc_xprt_ops xprt_rdma_procs;
158 
159 static void
160 xprt_rdma_format_addresses4(struct rpc_xprt *xprt, struct sockaddr *sap)
161 {
162         struct sockaddr_in *sin = (struct sockaddr_in *)sap;
163         char buf[20];
164 
165         snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));
166         xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);
167 
168         xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA;
169 }
170 
171 static void
172 xprt_rdma_format_addresses6(struct rpc_xprt *xprt, struct sockaddr *sap)
173 {
174         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap;
175         char buf[40];
176 
177         snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);
178         xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);
179 
180         xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA6;
181 }
182 
183 void
184 xprt_rdma_format_addresses(struct rpc_xprt *xprt, struct sockaddr *sap)
185 {
186         char buf[128];
187 
188         switch (sap->sa_family) {
189         case AF_INET:
190                 xprt_rdma_format_addresses4(xprt, sap);
191                 break;
192         case AF_INET6:
193                 xprt_rdma_format_addresses6(xprt, sap);
194                 break;
195         default:
196                 pr_err("rpcrdma: Unrecognized address family\n");
197                 return;
198         }
199 
200         (void)rpc_ntop(sap, buf, sizeof(buf));
201         xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL);
202 
203         snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap));
204         xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);
205 
206         snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap));
207         xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
208 
209         xprt->address_strings[RPC_DISPLAY_PROTO] = "rdma";
210 }
211 
212 void
213 xprt_rdma_free_addresses(struct rpc_xprt *xprt)
214 {
215         unsigned int i;
216 
217         for (i = 0; i < RPC_DISPLAY_MAX; i++)
218                 switch (i) {
219                 case RPC_DISPLAY_PROTO:
220                 case RPC_DISPLAY_NETID:
221                         continue;
222                 default:
223                         kfree(xprt->address_strings[i]);
224                 }
225 }
226 
227 /**
228  * xprt_rdma_connect_worker - establish connection in the background
229  * @work: worker thread context
230  *
231  * Requester holds the xprt's send lock to prevent activity on this
232  * transport while a fresh connection is being established. RPC tasks
233  * sleep on the xprt's pending queue waiting for connect to complete.
234  */
235 static void
236 xprt_rdma_connect_worker(struct work_struct *work)
237 {
238         struct rpcrdma_xprt *r_xprt = container_of(work, struct rpcrdma_xprt,
239                                                    rx_connect_worker.work);
240         struct rpc_xprt *xprt = &r_xprt->rx_xprt;
241         int rc;
242 
243         rc = rpcrdma_xprt_connect(r_xprt);
244         xprt_clear_connecting(xprt);
245         if (r_xprt->rx_ep && r_xprt->rx_ep->re_connect_status > 0) {
246                 xprt->connect_cookie++;
247                 xprt->stat.connect_count++;
248                 xprt->stat.connect_time += (long)jiffies -
249                                            xprt->stat.connect_start;
250                 xprt_set_connected(xprt);
251                 rc = -EAGAIN;
252         } else {
253                 /* Force a call to xprt_rdma_close to clean up */
254                 spin_lock(&xprt->transport_lock);
255                 set_bit(XPRT_CLOSE_WAIT, &xprt->state);
256                 spin_unlock(&xprt->transport_lock);
257         }
258         xprt_wake_pending_tasks(xprt, rc);
259 }
260 
261 /**
262  * xprt_rdma_inject_disconnect - inject a connection fault
263  * @xprt: transport context
264  *
265  * If @xprt is connected, disconnect it to simulate spurious connection
266  * loss.
267  */
268 static void
269 xprt_rdma_inject_disconnect(struct rpc_xprt *xprt)
270 {
271         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
272 
273         trace_xprtrdma_op_inject_dsc(r_xprt);
274         rdma_disconnect(r_xprt->rx_ep->re_id);
275 }
276 
277 /**
278  * xprt_rdma_destroy - Full tear down of transport
279  * @xprt: doomed transport context
280  *
281  * Caller guarantees there will be no more calls to us with
282  * this @xprt.
283  */
284 static void
285 xprt_rdma_destroy(struct rpc_xprt *xprt)
286 {
287         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
288 
289         trace_xprtrdma_op_destroy(r_xprt);
290 
291         cancel_delayed_work_sync(&r_xprt->rx_connect_worker);
292 
293         rpcrdma_xprt_disconnect(r_xprt);
294         rpcrdma_buffer_destroy(&r_xprt->rx_buf);
295 
296         xprt_rdma_free_addresses(xprt);
297         xprt_free(xprt);
298 
299         module_put(THIS_MODULE);
300 }
301 
302 /* 60 second timeout, no retries */
303 static const struct rpc_timeout xprt_rdma_default_timeout = {
304         .to_initval = 60 * HZ,
305         .to_maxval = 60 * HZ,
306 };
307 
308 /**
309  * xprt_setup_rdma - Set up transport to use RDMA
310  *
311  * @args: rpc transport arguments
312  */
313 static struct rpc_xprt *
314 xprt_setup_rdma(struct xprt_create *args)
315 {
316         struct rpc_xprt *xprt;
317         struct rpcrdma_xprt *new_xprt;
318         struct sockaddr *sap;
319         int rc;
320 
321         if (args->addrlen > sizeof(xprt->addr))
322                 return ERR_PTR(-EBADF);
323 
324         if (!try_module_get(THIS_MODULE))
325                 return ERR_PTR(-EIO);
326 
327         xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0,
328                           xprt_rdma_slot_table_entries);
329         if (!xprt) {
330                 module_put(THIS_MODULE);
331                 return ERR_PTR(-ENOMEM);
332         }
333 
334         xprt->timeout = &xprt_rdma_default_timeout;
335         xprt->connect_timeout = xprt->timeout->to_initval;
336         xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
337         xprt->bind_timeout = RPCRDMA_BIND_TO;
338         xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;
339         xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO;
340 
341         xprt->resvport = 0;             /* privileged port not needed */
342         xprt->ops = &xprt_rdma_procs;
343 
344         /*
345          * Set up RDMA-specific connect data.
346          */
347         sap = args->dstaddr;
348 
349         /* Ensure xprt->addr holds valid server TCP (not RDMA)
350          * address, for any side protocols which peek at it */
351         xprt->prot = IPPROTO_TCP;
352         xprt->addrlen = args->addrlen;
353         memcpy(&xprt->addr, sap, xprt->addrlen);
354 
355         if (rpc_get_port(sap))
356                 xprt_set_bound(xprt);
357         xprt_rdma_format_addresses(xprt, sap);
358 
359         new_xprt = rpcx_to_rdmax(xprt);
360         rc = rpcrdma_buffer_create(new_xprt);
361         if (rc) {
362                 xprt_rdma_free_addresses(xprt);
363                 xprt_free(xprt);
364                 module_put(THIS_MODULE);
365                 return ERR_PTR(rc);
366         }
367 
368         INIT_DELAYED_WORK(&new_xprt->rx_connect_worker,
369                           xprt_rdma_connect_worker);
370 
371         xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT;
372 
373         dprintk("RPC:       %s: %s:%s\n", __func__,
374                 xprt->address_strings[RPC_DISPLAY_ADDR],
375                 xprt->address_strings[RPC_DISPLAY_PORT]);
376         trace_xprtrdma_create(new_xprt);
377         return xprt;
378 }
379 
380 /**
381  * xprt_rdma_close - close a transport connection
382  * @xprt: transport context
383  *
384  * Called during autoclose or device removal.
385  *
386  * Caller holds @xprt's send lock to prevent activity on this
387  * transport while the connection is torn down.
388  */
389 void xprt_rdma_close(struct rpc_xprt *xprt)
390 {
391         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
392 
393         trace_xprtrdma_op_close(r_xprt);
394 
395         rpcrdma_xprt_disconnect(r_xprt);
396 
397         xprt->reestablish_timeout = 0;
398         ++xprt->connect_cookie;
399         xprt_disconnect_done(xprt);
400 }
401 
402 /**
403  * xprt_rdma_set_port - update server port with rpcbind result
404  * @xprt: controlling RPC transport
405  * @port: new port value
406  *
407  * Transport connect status is unchanged.
408  */
409 static void
410 xprt_rdma_set_port(struct rpc_xprt *xprt, u16 port)
411 {
412         struct sockaddr *sap = (struct sockaddr *)&xprt->addr;
413         char buf[8];
414 
415         rpc_set_port(sap, port);
416 
417         kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
418         snprintf(buf, sizeof(buf), "%u", port);
419         xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);
420 
421         kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
422         snprintf(buf, sizeof(buf), "%4hx", port);
423         xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
424 
425         trace_xprtrdma_op_setport(container_of(xprt, struct rpcrdma_xprt,
426                                                rx_xprt));
427 }
428 
429 /**
430  * xprt_rdma_timer - invoked when an RPC times out
431  * @xprt: controlling RPC transport
432  * @task: RPC task that timed out
433  *
434  * Invoked when the transport is still connected, but an RPC
435  * retransmit timeout occurs.
436  *
437  * Since RDMA connections don't have a keep-alive, forcibly
438  * disconnect and retry to connect. This drives full
439  * detection of the network path, and retransmissions of
440  * all pending RPCs.
441  */
442 static void
443 xprt_rdma_timer(struct rpc_xprt *xprt, struct rpc_task *task)
444 {
445         xprt_force_disconnect(xprt);
446 }
447 
448 /**
449  * xprt_rdma_set_connect_timeout - set timeouts for establishing a connection
450  * @xprt: controlling transport instance
451  * @connect_timeout: reconnect timeout after client disconnects
452  * @reconnect_timeout: reconnect timeout after server disconnects
453  *
454  */
455 static void xprt_rdma_set_connect_timeout(struct rpc_xprt *xprt,
456                                           unsigned long connect_timeout,
457                                           unsigned long reconnect_timeout)
458 {
459         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
460 
461         trace_xprtrdma_op_set_cto(r_xprt, connect_timeout, reconnect_timeout);
462 
463         spin_lock(&xprt->transport_lock);
464 
465         if (connect_timeout < xprt->connect_timeout) {
466                 struct rpc_timeout to;
467                 unsigned long initval;
468 
469                 to = *xprt->timeout;
470                 initval = connect_timeout;
471                 if (initval < RPCRDMA_INIT_REEST_TO << 1)
472                         initval = RPCRDMA_INIT_REEST_TO << 1;
473                 to.to_initval = initval;
474                 to.to_maxval = initval;
475                 r_xprt->rx_timeout = to;
476                 xprt->timeout = &r_xprt->rx_timeout;
477                 xprt->connect_timeout = connect_timeout;
478         }
479 
480         if (reconnect_timeout < xprt->max_reconnect_timeout)
481                 xprt->max_reconnect_timeout = reconnect_timeout;
482 
483         spin_unlock(&xprt->transport_lock);
484 }
485 
486 /**
487  * xprt_rdma_connect - schedule an attempt to reconnect
488  * @xprt: transport state
489  * @task: RPC scheduler context (unused)
490  *
491  */
492 static void
493 xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task)
494 {
495         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
496         struct rpcrdma_ep *ep = r_xprt->rx_ep;
497         unsigned long delay;
498 
499         delay = 0;
500         if (ep && ep->re_connect_status != 0) {
501                 delay = xprt_reconnect_delay(xprt);
502                 xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO);
503         }
504         trace_xprtrdma_op_connect(r_xprt, delay);
505         queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker,
506                            delay);
507 }
508 
509 /**
510  * xprt_rdma_alloc_slot - allocate an rpc_rqst
511  * @xprt: controlling RPC transport
512  * @task: RPC task requesting a fresh rpc_rqst
513  *
514  * tk_status values:
515  *      %0 if task->tk_rqstp points to a fresh rpc_rqst
516  *      %-EAGAIN if no rpc_rqst is available; queued on backlog
517  */
518 static void
519 xprt_rdma_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
520 {
521         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
522         struct rpcrdma_req *req;
523 
524         req = rpcrdma_buffer_get(&r_xprt->rx_buf);
525         if (!req)
526                 goto out_sleep;
527         task->tk_rqstp = &req->rl_slot;
528         task->tk_status = 0;
529         return;
530 
531 out_sleep:
532         set_bit(XPRT_CONGESTED, &xprt->state);
533         rpc_sleep_on(&xprt->backlog, task, NULL);
534         task->tk_status = -EAGAIN;
535 }
536 
537 /**
538  * xprt_rdma_free_slot - release an rpc_rqst
539  * @xprt: controlling RPC transport
540  * @rqst: rpc_rqst to release
541  *
542  */
543 static void
544 xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst)
545 {
546         struct rpcrdma_xprt *r_xprt =
547                 container_of(xprt, struct rpcrdma_xprt, rx_xprt);
548 
549         memset(rqst, 0, sizeof(*rqst));
550         rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst));
551         if (unlikely(!rpc_wake_up_next(&xprt->backlog)))
552                 clear_bit(XPRT_CONGESTED, &xprt->state);
553 }
554 
555 static bool rpcrdma_check_regbuf(struct rpcrdma_xprt *r_xprt,
556                                  struct rpcrdma_regbuf *rb, size_t size,
557                                  gfp_t flags)
558 {
559         if (unlikely(rdmab_length(rb) < size)) {
560                 if (!rpcrdma_regbuf_realloc(rb, size, flags))
561                         return false;
562                 r_xprt->rx_stats.hardway_register_count += size;
563         }
564         return true;
565 }
566 
567 /**
568  * xprt_rdma_allocate - allocate transport resources for an RPC
569  * @task: RPC task
570  *
571  * Return values:
572  *        0:    Success; rq_buffer points to RPC buffer to use
573  *   ENOMEM:    Out of memory, call again later
574  *      EIO:    A permanent error occurred, do not retry
575  */
576 static int
577 xprt_rdma_allocate(struct rpc_task *task)
578 {
579         struct rpc_rqst *rqst = task->tk_rqstp;
580         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
581         struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
582         gfp_t flags;
583 
584         flags = RPCRDMA_DEF_GFP;
585         if (RPC_IS_SWAPPER(task))
586                 flags = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN;
587 
588         if (!rpcrdma_check_regbuf(r_xprt, req->rl_sendbuf, rqst->rq_callsize,
589                                   flags))
590                 goto out_fail;
591         if (!rpcrdma_check_regbuf(r_xprt, req->rl_recvbuf, rqst->rq_rcvsize,
592                                   flags))
593                 goto out_fail;
594 
595         rqst->rq_buffer = rdmab_data(req->rl_sendbuf);
596         rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf);
597         trace_xprtrdma_op_allocate(task, req);
598         return 0;
599 
600 out_fail:
601         trace_xprtrdma_op_allocate(task, NULL);
602         return -ENOMEM;
603 }
604 
605 /**
606  * xprt_rdma_free - release resources allocated by xprt_rdma_allocate
607  * @task: RPC task
608  *
609  * Caller guarantees rqst->rq_buffer is non-NULL.
610  */
611 static void
612 xprt_rdma_free(struct rpc_task *task)
613 {
614         struct rpc_rqst *rqst = task->tk_rqstp;
615         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);
616         struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
617 
618         trace_xprtrdma_op_free(task, req);
619 
620         if (!list_empty(&req->rl_registered))
621                 frwr_unmap_sync(r_xprt, req);
622 
623         /* XXX: If the RPC is completing because of a signal and
624          * not because a reply was received, we ought to ensure
625          * that the Send completion has fired, so that memory
626          * involved with the Send is not still visible to the NIC.
627          */
628 }
629 
630 /**
631  * xprt_rdma_send_request - marshal and send an RPC request
632  * @rqst: RPC message in rq_snd_buf
633  *
634  * Caller holds the transport's write lock.
635  *
636  * Returns:
637  *      %0 if the RPC message has been sent
638  *      %-ENOTCONN if the caller should reconnect and call again
639  *      %-EAGAIN if the caller should call again
640  *      %-ENOBUFS if the caller should call again after a delay
641  *      %-EMSGSIZE if encoding ran out of buffer space. The request
642  *              was not sent. Do not try to send this message again.
643  *      %-EIO if an I/O error occurred. The request was not sent.
644  *              Do not try to send this message again.
645  */
646 static int
647 xprt_rdma_send_request(struct rpc_rqst *rqst)
648 {
649         struct rpc_xprt *xprt = rqst->rq_xprt;
650         struct rpcrdma_req *req = rpcr_to_rdmar(rqst);
651         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
652         int rc = 0;
653 
654 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
655         if (unlikely(!rqst->rq_buffer))
656                 return xprt_rdma_bc_send_reply(rqst);
657 #endif  /* CONFIG_SUNRPC_BACKCHANNEL */
658 
659         if (!xprt_connected(xprt))
660                 return -ENOTCONN;
661 
662         if (!xprt_request_get_cong(xprt, rqst))
663                 return -EBADSLT;
664 
665         rc = rpcrdma_marshal_req(r_xprt, rqst);
666         if (rc < 0)
667                 goto failed_marshal;
668 
669         /* Must suppress retransmit to maintain credits */
670         if (rqst->rq_connect_cookie == xprt->connect_cookie)
671                 goto drop_connection;
672         rqst->rq_xtime = ktime_get();
673 
674         if (rpcrdma_post_sends(r_xprt, req))
675                 goto drop_connection;
676 
677         rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len;
678 
679         /* An RPC with no reply will throw off credit accounting,
680          * so drop the connection to reset the credit grant.
681          */
682         if (!rpc_reply_expected(rqst->rq_task))
683                 goto drop_connection;
684         return 0;
685 
686 failed_marshal:
687         if (rc != -ENOTCONN)
688                 return rc;
689 drop_connection:
690         xprt_rdma_close(xprt);
691         return -ENOTCONN;
692 }
693 
694 void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
695 {
696         struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
697         long idle_time = 0;
698 
699         if (xprt_connected(xprt))
700                 idle_time = (long)(jiffies - xprt->last_used) / HZ;
701 
702         seq_puts(seq, "\txprt:\trdma ");
703         seq_printf(seq, "%u %lu %lu %lu %ld %lu %lu %lu %llu %llu ",
704                    0,   /* need a local port? */
705                    xprt->stat.bind_count,
706                    xprt->stat.connect_count,
707                    xprt->stat.connect_time / HZ,
708                    idle_time,
709                    xprt->stat.sends,
710                    xprt->stat.recvs,
711                    xprt->stat.bad_xids,
712                    xprt->stat.req_u,
713                    xprt->stat.bklog_u);
714         seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %lu %lu %lu %lu ",
715                    r_xprt->rx_stats.read_chunk_count,
716                    r_xprt->rx_stats.write_chunk_count,
717                    r_xprt->rx_stats.reply_chunk_count,
718                    r_xprt->rx_stats.total_rdma_request,
719                    r_xprt->rx_stats.total_rdma_reply,
720                    r_xprt->rx_stats.pullup_copy_count,
721                    r_xprt->rx_stats.fixup_copy_count,
722                    r_xprt->rx_stats.hardway_register_count,
723                    r_xprt->rx_stats.failed_marshal_count,
724                    r_xprt->rx_stats.bad_reply_count,
725                    r_xprt->rx_stats.nomsg_call_count);
726         seq_printf(seq, "%lu %lu %lu %lu %lu %lu\n",
727                    r_xprt->rx_stats.mrs_recycled,
728                    r_xprt->rx_stats.mrs_orphaned,
729                    r_xprt->rx_stats.mrs_allocated,
730                    r_xprt->rx_stats.local_inv_needed,
731                    r_xprt->rx_stats.empty_sendctx_q,
732                    r_xprt->rx_stats.reply_waits_for_send);
733 }
734 
735 static int
736 xprt_rdma_enable_swap(struct rpc_xprt *xprt)
737 {
738         return 0;
739 }
740 
741 static void
742 xprt_rdma_disable_swap(struct rpc_xprt *xprt)
743 {
744 }
745 
746 /*
747  * Plumbing for rpc transport switch and kernel module
748  */
749 
750 static const struct rpc_xprt_ops xprt_rdma_procs = {
751         .reserve_xprt           = xprt_reserve_xprt_cong,
752         .release_xprt           = xprt_release_xprt_cong, /* sunrpc/xprt.c */
753         .alloc_slot             = xprt_rdma_alloc_slot,
754         .free_slot              = xprt_rdma_free_slot,
755         .release_request        = xprt_release_rqst_cong,       /* ditto */
756         .wait_for_reply_request = xprt_wait_for_reply_request_def, /* ditto */
757         .timer                  = xprt_rdma_timer,
758         .rpcbind                = rpcb_getport_async,   /* sunrpc/rpcb_clnt.c */
759         .set_port               = xprt_rdma_set_port,
760         .connect                = xprt_rdma_connect,
761         .buf_alloc              = xprt_rdma_allocate,
762         .buf_free               = xprt_rdma_free,
763         .send_request           = xprt_rdma_send_request,
764         .close                  = xprt_rdma_close,
765         .destroy                = xprt_rdma_destroy,
766         .set_connect_timeout    = xprt_rdma_set_connect_timeout,
767         .print_stats            = xprt_rdma_print_stats,
768         .enable_swap            = xprt_rdma_enable_swap,
769         .disable_swap           = xprt_rdma_disable_swap,
770         .inject_disconnect      = xprt_rdma_inject_disconnect,
771 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
772         .bc_setup               = xprt_rdma_bc_setup,
773         .bc_maxpayload          = xprt_rdma_bc_maxpayload,
774         .bc_num_slots           = xprt_rdma_bc_max_slots,
775         .bc_free_rqst           = xprt_rdma_bc_free_rqst,
776         .bc_destroy             = xprt_rdma_bc_destroy,
777 #endif
778 };
779 
780 static struct xprt_class xprt_rdma = {
781         .list                   = LIST_HEAD_INIT(xprt_rdma.list),
782         .name                   = "rdma",
783         .owner                  = THIS_MODULE,
784         .ident                  = XPRT_TRANSPORT_RDMA,
785         .setup                  = xprt_setup_rdma,
786 };
787 
788 void xprt_rdma_cleanup(void)
789 {
790 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
791         if (sunrpc_table_header) {
792                 unregister_sysctl_table(sunrpc_table_header);
793                 sunrpc_table_header = NULL;
794         }
795 #endif
796 
797         xprt_unregister_transport(&xprt_rdma);
798         xprt_unregister_transport(&xprt_rdma_bc);
799 }
800 
801 int xprt_rdma_init(void)
802 {
803         int rc;
804 
805         rc = xprt_register_transport(&xprt_rdma);
806         if (rc)
807                 return rc;
808 
809         rc = xprt_register_transport(&xprt_rdma_bc);
810         if (rc) {
811                 xprt_unregister_transport(&xprt_rdma);
812                 return rc;
813         }
814 
815 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
816         if (!sunrpc_table_header)
817                 sunrpc_table_header = register_sysctl_table(sunrpc_table);
818 #endif
819         return 0;
820 }
821 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp