~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/block/blk-core.c

Version: ~ [ linux-5.13-rc5 ] ~ [ linux-5.12.9 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.42 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.124 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.193 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.235 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.271 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.271 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Copyright (C) 1991, 1992 Linus Torvalds
  4  * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
  5  * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
  6  * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
  7  * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
  8  *      -  July2000
  9  * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 10  */
 11 
 12 /*
 13  * This handles all read/write requests to block devices
 14  */
 15 #include <linux/kernel.h>
 16 #include <linux/module.h>
 17 #include <linux/backing-dev.h>
 18 #include <linux/bio.h>
 19 #include <linux/blkdev.h>
 20 #include <linux/blk-mq.h>
 21 #include <linux/highmem.h>
 22 #include <linux/mm.h>
 23 #include <linux/kernel_stat.h>
 24 #include <linux/string.h>
 25 #include <linux/init.h>
 26 #include <linux/completion.h>
 27 #include <linux/slab.h>
 28 #include <linux/swap.h>
 29 #include <linux/writeback.h>
 30 #include <linux/task_io_accounting_ops.h>
 31 #include <linux/fault-inject.h>
 32 #include <linux/list_sort.h>
 33 #include <linux/delay.h>
 34 #include <linux/ratelimit.h>
 35 #include <linux/pm_runtime.h>
 36 #include <linux/blk-cgroup.h>
 37 #include <linux/t10-pi.h>
 38 #include <linux/debugfs.h>
 39 #include <linux/bpf.h>
 40 #include <linux/psi.h>
 41 
 42 #define CREATE_TRACE_POINTS
 43 #include <trace/events/block.h>
 44 
 45 #include "blk.h"
 46 #include "blk-mq.h"
 47 #include "blk-mq-sched.h"
 48 #include "blk-pm.h"
 49 #include "blk-rq-qos.h"
 50 
 51 #ifdef CONFIG_DEBUG_FS
 52 struct dentry *blk_debugfs_root;
 53 #endif
 54 
 55 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
 56 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
 57 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
 58 EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
 59 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
 60 
 61 DEFINE_IDA(blk_queue_ida);
 62 
 63 /*
 64  * For queue allocation
 65  */
 66 struct kmem_cache *blk_requestq_cachep;
 67 
 68 /*
 69  * Controlling structure to kblockd
 70  */
 71 static struct workqueue_struct *kblockd_workqueue;
 72 
 73 /**
 74  * blk_queue_flag_set - atomically set a queue flag
 75  * @flag: flag to be set
 76  * @q: request queue
 77  */
 78 void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
 79 {
 80         set_bit(flag, &q->queue_flags);
 81 }
 82 EXPORT_SYMBOL(blk_queue_flag_set);
 83 
 84 /**
 85  * blk_queue_flag_clear - atomically clear a queue flag
 86  * @flag: flag to be cleared
 87  * @q: request queue
 88  */
 89 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
 90 {
 91         clear_bit(flag, &q->queue_flags);
 92 }
 93 EXPORT_SYMBOL(blk_queue_flag_clear);
 94 
 95 /**
 96  * blk_queue_flag_test_and_set - atomically test and set a queue flag
 97  * @flag: flag to be set
 98  * @q: request queue
 99  *
100  * Returns the previous value of @flag - 0 if the flag was not set and 1 if
101  * the flag was already set.
102  */
103 bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
104 {
105         return test_and_set_bit(flag, &q->queue_flags);
106 }
107 EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
108 
109 void blk_rq_init(struct request_queue *q, struct request *rq)
110 {
111         memset(rq, 0, sizeof(*rq));
112 
113         INIT_LIST_HEAD(&rq->queuelist);
114         rq->q = q;
115         rq->__sector = (sector_t) -1;
116         INIT_HLIST_NODE(&rq->hash);
117         RB_CLEAR_NODE(&rq->rb_node);
118         rq->tag = -1;
119         rq->internal_tag = -1;
120         rq->start_time_ns = ktime_get_ns();
121         rq->part = NULL;
122         refcount_set(&rq->ref, 1);
123 }
124 EXPORT_SYMBOL(blk_rq_init);
125 
126 #define REQ_OP_NAME(name) [REQ_OP_##name] = #name
127 static const char *const blk_op_name[] = {
128         REQ_OP_NAME(READ),
129         REQ_OP_NAME(WRITE),
130         REQ_OP_NAME(FLUSH),
131         REQ_OP_NAME(DISCARD),
132         REQ_OP_NAME(SECURE_ERASE),
133         REQ_OP_NAME(ZONE_RESET),
134         REQ_OP_NAME(ZONE_RESET_ALL),
135         REQ_OP_NAME(ZONE_OPEN),
136         REQ_OP_NAME(ZONE_CLOSE),
137         REQ_OP_NAME(ZONE_FINISH),
138         REQ_OP_NAME(WRITE_SAME),
139         REQ_OP_NAME(WRITE_ZEROES),
140         REQ_OP_NAME(SCSI_IN),
141         REQ_OP_NAME(SCSI_OUT),
142         REQ_OP_NAME(DRV_IN),
143         REQ_OP_NAME(DRV_OUT),
144 };
145 #undef REQ_OP_NAME
146 
147 /**
148  * blk_op_str - Return string XXX in the REQ_OP_XXX.
149  * @op: REQ_OP_XXX.
150  *
151  * Description: Centralize block layer function to convert REQ_OP_XXX into
152  * string format. Useful in the debugging and tracing bio or request. For
153  * invalid REQ_OP_XXX it returns string "UNKNOWN".
154  */
155 inline const char *blk_op_str(unsigned int op)
156 {
157         const char *op_str = "UNKNOWN";
158 
159         if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
160                 op_str = blk_op_name[op];
161 
162         return op_str;
163 }
164 EXPORT_SYMBOL_GPL(blk_op_str);
165 
166 static const struct {
167         int             errno;
168         const char      *name;
169 } blk_errors[] = {
170         [BLK_STS_OK]            = { 0,          "" },
171         [BLK_STS_NOTSUPP]       = { -EOPNOTSUPP, "operation not supported" },
172         [BLK_STS_TIMEOUT]       = { -ETIMEDOUT, "timeout" },
173         [BLK_STS_NOSPC]         = { -ENOSPC,    "critical space allocation" },
174         [BLK_STS_TRANSPORT]     = { -ENOLINK,   "recoverable transport" },
175         [BLK_STS_TARGET]        = { -EREMOTEIO, "critical target" },
176         [BLK_STS_NEXUS]         = { -EBADE,     "critical nexus" },
177         [BLK_STS_MEDIUM]        = { -ENODATA,   "critical medium" },
178         [BLK_STS_PROTECTION]    = { -EILSEQ,    "protection" },
179         [BLK_STS_RESOURCE]      = { -ENOMEM,    "kernel resource" },
180         [BLK_STS_DEV_RESOURCE]  = { -EBUSY,     "device resource" },
181         [BLK_STS_AGAIN]         = { -EAGAIN,    "nonblocking retry" },
182 
183         /* device mapper special case, should not leak out: */
184         [BLK_STS_DM_REQUEUE]    = { -EREMCHG, "dm internal retry" },
185 
186         /* everything else not covered above: */
187         [BLK_STS_IOERR]         = { -EIO,       "I/O" },
188 };
189 
190 blk_status_t errno_to_blk_status(int errno)
191 {
192         int i;
193 
194         for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
195                 if (blk_errors[i].errno == errno)
196                         return (__force blk_status_t)i;
197         }
198 
199         return BLK_STS_IOERR;
200 }
201 EXPORT_SYMBOL_GPL(errno_to_blk_status);
202 
203 int blk_status_to_errno(blk_status_t status)
204 {
205         int idx = (__force int)status;
206 
207         if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
208                 return -EIO;
209         return blk_errors[idx].errno;
210 }
211 EXPORT_SYMBOL_GPL(blk_status_to_errno);
212 
213 static void print_req_error(struct request *req, blk_status_t status,
214                 const char *caller)
215 {
216         int idx = (__force int)status;
217 
218         if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
219                 return;
220 
221         printk_ratelimited(KERN_ERR
222                 "%s: %s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
223                 "phys_seg %u prio class %u\n",
224                 caller, blk_errors[idx].name,
225                 req->rq_disk ? req->rq_disk->disk_name : "?",
226                 blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)),
227                 req->cmd_flags & ~REQ_OP_MASK,
228                 req->nr_phys_segments,
229                 IOPRIO_PRIO_CLASS(req->ioprio));
230 }
231 
232 static void req_bio_endio(struct request *rq, struct bio *bio,
233                           unsigned int nbytes, blk_status_t error)
234 {
235         if (error)
236                 bio->bi_status = error;
237 
238         if (unlikely(rq->rq_flags & RQF_QUIET))
239                 bio_set_flag(bio, BIO_QUIET);
240 
241         bio_advance(bio, nbytes);
242 
243         /* don't actually finish bio if it's part of flush sequence */
244         if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
245                 bio_endio(bio);
246 }
247 
248 void blk_dump_rq_flags(struct request *rq, char *msg)
249 {
250         printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
251                 rq->rq_disk ? rq->rq_disk->disk_name : "?",
252                 (unsigned long long) rq->cmd_flags);
253 
254         printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
255                (unsigned long long)blk_rq_pos(rq),
256                blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
257         printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
258                rq->bio, rq->biotail, blk_rq_bytes(rq));
259 }
260 EXPORT_SYMBOL(blk_dump_rq_flags);
261 
262 /**
263  * blk_sync_queue - cancel any pending callbacks on a queue
264  * @q: the queue
265  *
266  * Description:
267  *     The block layer may perform asynchronous callback activity
268  *     on a queue, such as calling the unplug function after a timeout.
269  *     A block device may call blk_sync_queue to ensure that any
270  *     such activity is cancelled, thus allowing it to release resources
271  *     that the callbacks might use. The caller must already have made sure
272  *     that its ->make_request_fn will not re-add plugging prior to calling
273  *     this function.
274  *
275  *     This function does not cancel any asynchronous activity arising
276  *     out of elevator or throttling code. That would require elevator_exit()
277  *     and blkcg_exit_queue() to be called with queue lock initialized.
278  *
279  */
280 void blk_sync_queue(struct request_queue *q)
281 {
282         del_timer_sync(&q->timeout);
283         cancel_work_sync(&q->timeout_work);
284 }
285 EXPORT_SYMBOL(blk_sync_queue);
286 
287 /**
288  * blk_set_pm_only - increment pm_only counter
289  * @q: request queue pointer
290  */
291 void blk_set_pm_only(struct request_queue *q)
292 {
293         atomic_inc(&q->pm_only);
294 }
295 EXPORT_SYMBOL_GPL(blk_set_pm_only);
296 
297 void blk_clear_pm_only(struct request_queue *q)
298 {
299         int pm_only;
300 
301         pm_only = atomic_dec_return(&q->pm_only);
302         WARN_ON_ONCE(pm_only < 0);
303         if (pm_only == 0)
304                 wake_up_all(&q->mq_freeze_wq);
305 }
306 EXPORT_SYMBOL_GPL(blk_clear_pm_only);
307 
308 void blk_put_queue(struct request_queue *q)
309 {
310         kobject_put(&q->kobj);
311 }
312 EXPORT_SYMBOL(blk_put_queue);
313 
314 void blk_set_queue_dying(struct request_queue *q)
315 {
316         blk_queue_flag_set(QUEUE_FLAG_DYING, q);
317 
318         /*
319          * When queue DYING flag is set, we need to block new req
320          * entering queue, so we call blk_freeze_queue_start() to
321          * prevent I/O from crossing blk_queue_enter().
322          */
323         blk_freeze_queue_start(q);
324 
325         if (queue_is_mq(q))
326                 blk_mq_wake_waiters(q);
327 
328         /* Make blk_queue_enter() reexamine the DYING flag. */
329         wake_up_all(&q->mq_freeze_wq);
330 }
331 EXPORT_SYMBOL_GPL(blk_set_queue_dying);
332 
333 /**
334  * blk_cleanup_queue - shutdown a request queue
335  * @q: request queue to shutdown
336  *
337  * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
338  * put it.  All future requests will be failed immediately with -ENODEV.
339  */
340 void blk_cleanup_queue(struct request_queue *q)
341 {
342         WARN_ON_ONCE(blk_queue_registered(q));
343 
344         /* mark @q DYING, no new request or merges will be allowed afterwards */
345         blk_set_queue_dying(q);
346 
347         blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
348         blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
349         blk_queue_flag_set(QUEUE_FLAG_DYING, q);
350 
351         /*
352          * Drain all requests queued before DYING marking. Set DEAD flag to
353          * prevent that blk_mq_run_hw_queues() accesses the hardware queues
354          * after draining finished.
355          */
356         blk_freeze_queue(q);
357 
358         rq_qos_exit(q);
359 
360         blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
361 
362         /* for synchronous bio-based driver finish in-flight integrity i/o */
363         blk_flush_integrity();
364 
365         /* @q won't process any more request, flush async actions */
366         del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
367         blk_sync_queue(q);
368 
369         if (queue_is_mq(q))
370                 blk_mq_exit_queue(q);
371 
372         /*
373          * In theory, request pool of sched_tags belongs to request queue.
374          * However, the current implementation requires tag_set for freeing
375          * requests, so free the pool now.
376          *
377          * Queue has become frozen, there can't be any in-queue requests, so
378          * it is safe to free requests now.
379          */
380         mutex_lock(&q->sysfs_lock);
381         if (q->elevator)
382                 blk_mq_sched_free_requests(q);
383         mutex_unlock(&q->sysfs_lock);
384 
385         percpu_ref_exit(&q->q_usage_counter);
386 
387         /* @q is and will stay empty, shutdown and put */
388         blk_put_queue(q);
389 }
390 EXPORT_SYMBOL(blk_cleanup_queue);
391 
392 struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
393 {
394         return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
395 }
396 EXPORT_SYMBOL(blk_alloc_queue);
397 
398 /**
399  * blk_queue_enter() - try to increase q->q_usage_counter
400  * @q: request queue pointer
401  * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT
402  */
403 int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
404 {
405         const bool pm = flags & BLK_MQ_REQ_PREEMPT;
406 
407         while (true) {
408                 bool success = false;
409 
410                 rcu_read_lock();
411                 if (percpu_ref_tryget_live(&q->q_usage_counter)) {
412                         /*
413                          * The code that increments the pm_only counter is
414                          * responsible for ensuring that that counter is
415                          * globally visible before the queue is unfrozen.
416                          */
417                         if (pm || !blk_queue_pm_only(q)) {
418                                 success = true;
419                         } else {
420                                 percpu_ref_put(&q->q_usage_counter);
421                         }
422                 }
423                 rcu_read_unlock();
424 
425                 if (success)
426                         return 0;
427 
428                 if (flags & BLK_MQ_REQ_NOWAIT)
429                         return -EBUSY;
430 
431                 /*
432                  * read pair of barrier in blk_freeze_queue_start(),
433                  * we need to order reading __PERCPU_REF_DEAD flag of
434                  * .q_usage_counter and reading .mq_freeze_depth or
435                  * queue dying flag, otherwise the following wait may
436                  * never return if the two reads are reordered.
437                  */
438                 smp_rmb();
439 
440                 wait_event(q->mq_freeze_wq,
441                            (!q->mq_freeze_depth &&
442                             (pm || (blk_pm_request_resume(q),
443                                     !blk_queue_pm_only(q)))) ||
444                            blk_queue_dying(q));
445                 if (blk_queue_dying(q))
446                         return -ENODEV;
447         }
448 }
449 
450 void blk_queue_exit(struct request_queue *q)
451 {
452         percpu_ref_put(&q->q_usage_counter);
453 }
454 
455 static void blk_queue_usage_counter_release(struct percpu_ref *ref)
456 {
457         struct request_queue *q =
458                 container_of(ref, struct request_queue, q_usage_counter);
459 
460         wake_up_all(&q->mq_freeze_wq);
461 }
462 
463 static void blk_rq_timed_out_timer(struct timer_list *t)
464 {
465         struct request_queue *q = from_timer(q, t, timeout);
466 
467         kblockd_schedule_work(&q->timeout_work);
468 }
469 
470 static void blk_timeout_work(struct work_struct *work)
471 {
472 }
473 
474 /**
475  * blk_alloc_queue_node - allocate a request queue
476  * @gfp_mask: memory allocation flags
477  * @node_id: NUMA node to allocate memory from
478  */
479 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
480 {
481         struct request_queue *q;
482         int ret;
483 
484         q = kmem_cache_alloc_node(blk_requestq_cachep,
485                                 gfp_mask | __GFP_ZERO, node_id);
486         if (!q)
487                 return NULL;
488 
489         q->last_merge = NULL;
490 
491         q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
492         if (q->id < 0)
493                 goto fail_q;
494 
495         ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
496         if (ret)
497                 goto fail_id;
498 
499         q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
500         if (!q->backing_dev_info)
501                 goto fail_split;
502 
503         q->stats = blk_alloc_queue_stats();
504         if (!q->stats)
505                 goto fail_stats;
506 
507         q->backing_dev_info->ra_pages = VM_READAHEAD_PAGES;
508         q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
509         q->backing_dev_info->name = "block";
510         q->node = node_id;
511 
512         timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
513                     laptop_mode_timer_fn, 0);
514         timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
515         INIT_WORK(&q->timeout_work, blk_timeout_work);
516         INIT_LIST_HEAD(&q->icq_list);
517 #ifdef CONFIG_BLK_CGROUP
518         INIT_LIST_HEAD(&q->blkg_list);
519 #endif
520 
521         kobject_init(&q->kobj, &blk_queue_ktype);
522 
523 #ifdef CONFIG_BLK_DEV_IO_TRACE
524         mutex_init(&q->blk_trace_mutex);
525 #endif
526         mutex_init(&q->sysfs_lock);
527         mutex_init(&q->sysfs_dir_lock);
528         spin_lock_init(&q->queue_lock);
529 
530         init_waitqueue_head(&q->mq_freeze_wq);
531         mutex_init(&q->mq_freeze_lock);
532 
533         /*
534          * Init percpu_ref in atomic mode so that it's faster to shutdown.
535          * See blk_register_queue() for details.
536          */
537         if (percpu_ref_init(&q->q_usage_counter,
538                                 blk_queue_usage_counter_release,
539                                 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
540                 goto fail_bdi;
541 
542         if (blkcg_init_queue(q))
543                 goto fail_ref;
544 
545         return q;
546 
547 fail_ref:
548         percpu_ref_exit(&q->q_usage_counter);
549 fail_bdi:
550         blk_free_queue_stats(q->stats);
551 fail_stats:
552         bdi_put(q->backing_dev_info);
553 fail_split:
554         bioset_exit(&q->bio_split);
555 fail_id:
556         ida_simple_remove(&blk_queue_ida, q->id);
557 fail_q:
558         kmem_cache_free(blk_requestq_cachep, q);
559         return NULL;
560 }
561 EXPORT_SYMBOL(blk_alloc_queue_node);
562 
563 bool blk_get_queue(struct request_queue *q)
564 {
565         if (likely(!blk_queue_dying(q))) {
566                 __blk_get_queue(q);
567                 return true;
568         }
569 
570         return false;
571 }
572 EXPORT_SYMBOL(blk_get_queue);
573 
574 /**
575  * blk_get_request - allocate a request
576  * @q: request queue to allocate a request for
577  * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
578  * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
579  */
580 struct request *blk_get_request(struct request_queue *q, unsigned int op,
581                                 blk_mq_req_flags_t flags)
582 {
583         struct request *req;
584 
585         WARN_ON_ONCE(op & REQ_NOWAIT);
586         WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
587 
588         req = blk_mq_alloc_request(q, op, flags);
589         if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
590                 q->mq_ops->initialize_rq_fn(req);
591 
592         return req;
593 }
594 EXPORT_SYMBOL(blk_get_request);
595 
596 void blk_put_request(struct request *req)
597 {
598         blk_mq_free_request(req);
599 }
600 EXPORT_SYMBOL(blk_put_request);
601 
602 bool bio_attempt_back_merge(struct request *req, struct bio *bio,
603                 unsigned int nr_segs)
604 {
605         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
606 
607         if (!ll_back_merge_fn(req, bio, nr_segs))
608                 return false;
609 
610         trace_block_bio_backmerge(req->q, req, bio);
611         rq_qos_merge(req->q, req, bio);
612 
613         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
614                 blk_rq_set_mixed_merge(req);
615 
616         req->biotail->bi_next = bio;
617         req->biotail = bio;
618         req->__data_len += bio->bi_iter.bi_size;
619 
620         blk_account_io_start(req, false);
621         return true;
622 }
623 
624 bool bio_attempt_front_merge(struct request *req, struct bio *bio,
625                 unsigned int nr_segs)
626 {
627         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
628 
629         if (!ll_front_merge_fn(req, bio, nr_segs))
630                 return false;
631 
632         trace_block_bio_frontmerge(req->q, req, bio);
633         rq_qos_merge(req->q, req, bio);
634 
635         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
636                 blk_rq_set_mixed_merge(req);
637 
638         bio->bi_next = req->bio;
639         req->bio = bio;
640 
641         req->__sector = bio->bi_iter.bi_sector;
642         req->__data_len += bio->bi_iter.bi_size;
643 
644         blk_account_io_start(req, false);
645         return true;
646 }
647 
648 bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
649                 struct bio *bio)
650 {
651         unsigned short segments = blk_rq_nr_discard_segments(req);
652 
653         if (segments >= queue_max_discard_segments(q))
654                 goto no_merge;
655         if (blk_rq_sectors(req) + bio_sectors(bio) >
656             blk_rq_get_max_sectors(req, blk_rq_pos(req)))
657                 goto no_merge;
658 
659         rq_qos_merge(q, req, bio);
660 
661         req->biotail->bi_next = bio;
662         req->biotail = bio;
663         req->__data_len += bio->bi_iter.bi_size;
664         req->nr_phys_segments = segments + 1;
665 
666         blk_account_io_start(req, false);
667         return true;
668 no_merge:
669         req_set_nomerge(q, req);
670         return false;
671 }
672 
673 /**
674  * blk_attempt_plug_merge - try to merge with %current's plugged list
675  * @q: request_queue new bio is being queued at
676  * @bio: new bio being queued
677  * @nr_segs: number of segments in @bio
678  * @same_queue_rq: pointer to &struct request that gets filled in when
679  * another request associated with @q is found on the plug list
680  * (optional, may be %NULL)
681  *
682  * Determine whether @bio being queued on @q can be merged with a request
683  * on %current's plugged list.  Returns %true if merge was successful,
684  * otherwise %false.
685  *
686  * Plugging coalesces IOs from the same issuer for the same purpose without
687  * going through @q->queue_lock.  As such it's more of an issuing mechanism
688  * than scheduling, and the request, while may have elvpriv data, is not
689  * added on the elevator at this point.  In addition, we don't have
690  * reliable access to the elevator outside queue lock.  Only check basic
691  * merging parameters without querying the elevator.
692  *
693  * Caller must ensure !blk_queue_nomerges(q) beforehand.
694  */
695 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
696                 unsigned int nr_segs, struct request **same_queue_rq)
697 {
698         struct blk_plug *plug;
699         struct request *rq;
700         struct list_head *plug_list;
701 
702         plug = blk_mq_plug(q, bio);
703         if (!plug)
704                 return false;
705 
706         plug_list = &plug->mq_list;
707 
708         list_for_each_entry_reverse(rq, plug_list, queuelist) {
709                 bool merged = false;
710 
711                 if (rq->q == q && same_queue_rq) {
712                         /*
713                          * Only blk-mq multiple hardware queues case checks the
714                          * rq in the same queue, there should be only one such
715                          * rq in a queue
716                          **/
717                         *same_queue_rq = rq;
718                 }
719 
720                 if (rq->q != q || !blk_rq_merge_ok(rq, bio))
721                         continue;
722 
723                 switch (blk_try_merge(rq, bio)) {
724                 case ELEVATOR_BACK_MERGE:
725                         merged = bio_attempt_back_merge(rq, bio, nr_segs);
726                         break;
727                 case ELEVATOR_FRONT_MERGE:
728                         merged = bio_attempt_front_merge(rq, bio, nr_segs);
729                         break;
730                 case ELEVATOR_DISCARD_MERGE:
731                         merged = bio_attempt_discard_merge(q, rq, bio);
732                         break;
733                 default:
734                         break;
735                 }
736 
737                 if (merged)
738                         return true;
739         }
740 
741         return false;
742 }
743 
744 static void handle_bad_sector(struct bio *bio, sector_t maxsector)
745 {
746         char b[BDEVNAME_SIZE];
747 
748         printk(KERN_INFO "attempt to access beyond end of device\n");
749         printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
750                         bio_devname(bio, b), bio->bi_opf,
751                         (unsigned long long)bio_end_sector(bio),
752                         (long long)maxsector);
753 }
754 
755 #ifdef CONFIG_FAIL_MAKE_REQUEST
756 
757 static DECLARE_FAULT_ATTR(fail_make_request);
758 
759 static int __init setup_fail_make_request(char *str)
760 {
761         return setup_fault_attr(&fail_make_request, str);
762 }
763 __setup("fail_make_request=", setup_fail_make_request);
764 
765 static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
766 {
767         return part->make_it_fail && should_fail(&fail_make_request, bytes);
768 }
769 
770 static int __init fail_make_request_debugfs(void)
771 {
772         struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
773                                                 NULL, &fail_make_request);
774 
775         return PTR_ERR_OR_ZERO(dir);
776 }
777 
778 late_initcall(fail_make_request_debugfs);
779 
780 #else /* CONFIG_FAIL_MAKE_REQUEST */
781 
782 static inline bool should_fail_request(struct hd_struct *part,
783                                         unsigned int bytes)
784 {
785         return false;
786 }
787 
788 #endif /* CONFIG_FAIL_MAKE_REQUEST */
789 
790 static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
791 {
792         const int op = bio_op(bio);
793 
794         if (part->policy && op_is_write(op)) {
795                 char b[BDEVNAME_SIZE];
796 
797                 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
798                         return false;
799 
800                 WARN_ONCE(1,
801                        "generic_make_request: Trying to write "
802                         "to read-only block-device %s (partno %d)\n",
803                         bio_devname(bio, b), part->partno);
804                 /* Older lvm-tools actually trigger this */
805                 return false;
806         }
807 
808         return false;
809 }
810 
811 static noinline int should_fail_bio(struct bio *bio)
812 {
813         if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
814                 return -EIO;
815         return 0;
816 }
817 ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
818 
819 /*
820  * Check whether this bio extends beyond the end of the device or partition.
821  * This may well happen - the kernel calls bread() without checking the size of
822  * the device, e.g., when mounting a file system.
823  */
824 static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
825 {
826         unsigned int nr_sectors = bio_sectors(bio);
827 
828         if (nr_sectors && maxsector &&
829             (nr_sectors > maxsector ||
830              bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
831                 handle_bad_sector(bio, maxsector);
832                 return -EIO;
833         }
834         return 0;
835 }
836 
837 /*
838  * Remap block n of partition p to block n+start(p) of the disk.
839  */
840 static inline int blk_partition_remap(struct bio *bio)
841 {
842         struct hd_struct *p;
843         int ret = -EIO;
844 
845         rcu_read_lock();
846         p = __disk_get_part(bio->bi_disk, bio->bi_partno);
847         if (unlikely(!p))
848                 goto out;
849         if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
850                 goto out;
851         if (unlikely(bio_check_ro(bio, p)))
852                 goto out;
853 
854         if (bio_sectors(bio)) {
855                 if (bio_check_eod(bio, part_nr_sects_read(p)))
856                         goto out;
857                 bio->bi_iter.bi_sector += p->start_sect;
858                 trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
859                                       bio->bi_iter.bi_sector - p->start_sect);
860         }
861         bio->bi_partno = 0;
862         ret = 0;
863 out:
864         rcu_read_unlock();
865         return ret;
866 }
867 
868 static noinline_for_stack bool
869 generic_make_request_checks(struct bio *bio)
870 {
871         struct request_queue *q;
872         int nr_sectors = bio_sectors(bio);
873         blk_status_t status = BLK_STS_IOERR;
874         char b[BDEVNAME_SIZE];
875 
876         might_sleep();
877 
878         q = bio->bi_disk->queue;
879         if (unlikely(!q)) {
880                 printk(KERN_ERR
881                        "generic_make_request: Trying to access "
882                         "nonexistent block-device %s (%Lu)\n",
883                         bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
884                 goto end_io;
885         }
886 
887         /*
888          * For a REQ_NOWAIT based request, return -EOPNOTSUPP
889          * if queue is not a request based queue.
890          */
891         if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q))
892                 goto not_supported;
893 
894         if (should_fail_bio(bio))
895                 goto end_io;
896 
897         if (bio->bi_partno) {
898                 if (unlikely(blk_partition_remap(bio)))
899                         goto end_io;
900         } else {
901                 if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
902                         goto end_io;
903                 if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
904                         goto end_io;
905         }
906 
907         /*
908          * Filter flush bio's early so that make_request based
909          * drivers without flush support don't have to worry
910          * about them.
911          */
912         if (op_is_flush(bio->bi_opf) &&
913             !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
914                 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
915                 if (!nr_sectors) {
916                         status = BLK_STS_OK;
917                         goto end_io;
918                 }
919         }
920 
921         if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
922                 bio->bi_opf &= ~REQ_HIPRI;
923 
924         switch (bio_op(bio)) {
925         case REQ_OP_DISCARD:
926                 if (!blk_queue_discard(q))
927                         goto not_supported;
928                 break;
929         case REQ_OP_SECURE_ERASE:
930                 if (!blk_queue_secure_erase(q))
931                         goto not_supported;
932                 break;
933         case REQ_OP_WRITE_SAME:
934                 if (!q->limits.max_write_same_sectors)
935                         goto not_supported;
936                 break;
937         case REQ_OP_ZONE_RESET:
938         case REQ_OP_ZONE_OPEN:
939         case REQ_OP_ZONE_CLOSE:
940         case REQ_OP_ZONE_FINISH:
941                 if (!blk_queue_is_zoned(q))
942                         goto not_supported;
943                 break;
944         case REQ_OP_ZONE_RESET_ALL:
945                 if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q))
946                         goto not_supported;
947                 break;
948         case REQ_OP_WRITE_ZEROES:
949                 if (!q->limits.max_write_zeroes_sectors)
950                         goto not_supported;
951                 break;
952         default:
953                 break;
954         }
955 
956         /*
957          * Various block parts want %current->io_context and lazy ioc
958          * allocation ends up trading a lot of pain for a small amount of
959          * memory.  Just allocate it upfront.  This may fail and block
960          * layer knows how to live with it.
961          */
962         create_io_context(GFP_ATOMIC, q->node);
963 
964         if (!blkcg_bio_issue_check(q, bio))
965                 return false;
966 
967         if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
968                 trace_block_bio_queue(q, bio);
969                 /* Now that enqueuing has been traced, we need to trace
970                  * completion as well.
971                  */
972                 bio_set_flag(bio, BIO_TRACE_COMPLETION);
973         }
974         return true;
975 
976 not_supported:
977         status = BLK_STS_NOTSUPP;
978 end_io:
979         bio->bi_status = status;
980         bio_endio(bio);
981         return false;
982 }
983 
984 /**
985  * generic_make_request - hand a buffer to its device driver for I/O
986  * @bio:  The bio describing the location in memory and on the device.
987  *
988  * generic_make_request() is used to make I/O requests of block
989  * devices. It is passed a &struct bio, which describes the I/O that needs
990  * to be done.
991  *
992  * generic_make_request() does not return any status.  The
993  * success/failure status of the request, along with notification of
994  * completion, is delivered asynchronously through the bio->bi_end_io
995  * function described (one day) else where.
996  *
997  * The caller of generic_make_request must make sure that bi_io_vec
998  * are set to describe the memory buffer, and that bi_dev and bi_sector are
999  * set to describe the device address, and the
1000  * bi_end_io and optionally bi_private are set to describe how
1001  * completion notification should be signaled.
1002  *
1003  * generic_make_request and the drivers it calls may use bi_next if this
1004  * bio happens to be merged with someone else, and may resubmit the bio to
1005  * a lower device by calling into generic_make_request recursively, which
1006  * means the bio should NOT be touched after the call to ->make_request_fn.
1007  */
1008 blk_qc_t generic_make_request(struct bio *bio)
1009 {
1010         /*
1011          * bio_list_on_stack[0] contains bios submitted by the current
1012          * make_request_fn.
1013          * bio_list_on_stack[1] contains bios that were submitted before
1014          * the current make_request_fn, but that haven't been processed
1015          * yet.
1016          */
1017         struct bio_list bio_list_on_stack[2];
1018         blk_qc_t ret = BLK_QC_T_NONE;
1019 
1020         if (!generic_make_request_checks(bio))
1021                 goto out;
1022 
1023         /*
1024          * We only want one ->make_request_fn to be active at a time, else
1025          * stack usage with stacked devices could be a problem.  So use
1026          * current->bio_list to keep a list of requests submited by a
1027          * make_request_fn function.  current->bio_list is also used as a
1028          * flag to say if generic_make_request is currently active in this
1029          * task or not.  If it is NULL, then no make_request is active.  If
1030          * it is non-NULL, then a make_request is active, and new requests
1031          * should be added at the tail
1032          */
1033         if (current->bio_list) {
1034                 bio_list_add(&current->bio_list[0], bio);
1035                 goto out;
1036         }
1037 
1038         /* following loop may be a bit non-obvious, and so deserves some
1039          * explanation.
1040          * Before entering the loop, bio->bi_next is NULL (as all callers
1041          * ensure that) so we have a list with a single bio.
1042          * We pretend that we have just taken it off a longer list, so
1043          * we assign bio_list to a pointer to the bio_list_on_stack,
1044          * thus initialising the bio_list of new bios to be
1045          * added.  ->make_request() may indeed add some more bios
1046          * through a recursive call to generic_make_request.  If it
1047          * did, we find a non-NULL value in bio_list and re-enter the loop
1048          * from the top.  In this case we really did just take the bio
1049          * of the top of the list (no pretending) and so remove it from
1050          * bio_list, and call into ->make_request() again.
1051          */
1052         BUG_ON(bio->bi_next);
1053         bio_list_init(&bio_list_on_stack[0]);
1054         current->bio_list = bio_list_on_stack;
1055         do {
1056                 struct request_queue *q = bio->bi_disk->queue;
1057                 blk_mq_req_flags_t flags = bio->bi_opf & REQ_NOWAIT ?
1058                         BLK_MQ_REQ_NOWAIT : 0;
1059 
1060                 if (likely(blk_queue_enter(q, flags) == 0)) {
1061                         struct bio_list lower, same;
1062 
1063                         /* Create a fresh bio_list for all subordinate requests */
1064                         bio_list_on_stack[1] = bio_list_on_stack[0];
1065                         bio_list_init(&bio_list_on_stack[0]);
1066                         ret = q->make_request_fn(q, bio);
1067 
1068                         blk_queue_exit(q);
1069 
1070                         /* sort new bios into those for a lower level
1071                          * and those for the same level
1072                          */
1073                         bio_list_init(&lower);
1074                         bio_list_init(&same);
1075                         while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
1076                                 if (q == bio->bi_disk->queue)
1077                                         bio_list_add(&same, bio);
1078                                 else
1079                                         bio_list_add(&lower, bio);
1080                         /* now assemble so we handle the lowest level first */
1081                         bio_list_merge(&bio_list_on_stack[0], &lower);
1082                         bio_list_merge(&bio_list_on_stack[0], &same);
1083                         bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
1084                 } else {
1085                         if (unlikely(!blk_queue_dying(q) &&
1086                                         (bio->bi_opf & REQ_NOWAIT)))
1087                                 bio_wouldblock_error(bio);
1088                         else
1089                                 bio_io_error(bio);
1090                 }
1091                 bio = bio_list_pop(&bio_list_on_stack[0]);
1092         } while (bio);
1093         current->bio_list = NULL; /* deactivate */
1094 
1095 out:
1096         return ret;
1097 }
1098 EXPORT_SYMBOL(generic_make_request);
1099 
1100 /**
1101  * direct_make_request - hand a buffer directly to its device driver for I/O
1102  * @bio:  The bio describing the location in memory and on the device.
1103  *
1104  * This function behaves like generic_make_request(), but does not protect
1105  * against recursion.  Must only be used if the called driver is known
1106  * to not call generic_make_request (or direct_make_request) again from
1107  * its make_request function.  (Calling direct_make_request again from
1108  * a workqueue is perfectly fine as that doesn't recurse).
1109  */
1110 blk_qc_t direct_make_request(struct bio *bio)
1111 {
1112         struct request_queue *q = bio->bi_disk->queue;
1113         bool nowait = bio->bi_opf & REQ_NOWAIT;
1114         blk_qc_t ret;
1115 
1116         if (!generic_make_request_checks(bio))
1117                 return BLK_QC_T_NONE;
1118 
1119         if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
1120                 if (nowait && !blk_queue_dying(q))
1121                         bio->bi_status = BLK_STS_AGAIN;
1122                 else
1123                         bio->bi_status = BLK_STS_IOERR;
1124                 bio_endio(bio);
1125                 return BLK_QC_T_NONE;
1126         }
1127 
1128         ret = q->make_request_fn(q, bio);
1129         blk_queue_exit(q);
1130         return ret;
1131 }
1132 EXPORT_SYMBOL_GPL(direct_make_request);
1133 
1134 /**
1135  * submit_bio - submit a bio to the block device layer for I/O
1136  * @bio: The &struct bio which describes the I/O
1137  *
1138  * submit_bio() is very similar in purpose to generic_make_request(), and
1139  * uses that function to do most of the work. Both are fairly rough
1140  * interfaces; @bio must be presetup and ready for I/O.
1141  *
1142  */
1143 blk_qc_t submit_bio(struct bio *bio)
1144 {
1145         bool workingset_read = false;
1146         unsigned long pflags;
1147         blk_qc_t ret;
1148 
1149         if (blkcg_punt_bio_submit(bio))
1150                 return BLK_QC_T_NONE;
1151 
1152         /*
1153          * If it's a regular read/write or a barrier with data attached,
1154          * go through the normal accounting stuff before submission.
1155          */
1156         if (bio_has_data(bio)) {
1157                 unsigned int count;
1158 
1159                 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
1160                         count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
1161                 else
1162                         count = bio_sectors(bio);
1163 
1164                 if (op_is_write(bio_op(bio))) {
1165                         count_vm_events(PGPGOUT, count);
1166                 } else {
1167                         if (bio_flagged(bio, BIO_WORKINGSET))
1168                                 workingset_read = true;
1169                         task_io_account_read(bio->bi_iter.bi_size);
1170                         count_vm_events(PGPGIN, count);
1171                 }
1172 
1173                 if (unlikely(block_dump)) {
1174                         char b[BDEVNAME_SIZE];
1175                         printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1176                         current->comm, task_pid_nr(current),
1177                                 op_is_write(bio_op(bio)) ? "WRITE" : "READ",
1178                                 (unsigned long long)bio->bi_iter.bi_sector,
1179                                 bio_devname(bio, b), count);
1180                 }
1181         }
1182 
1183         /*
1184          * If we're reading data that is part of the userspace
1185          * workingset, count submission time as memory stall. When the
1186          * device is congested, or the submitting cgroup IO-throttled,
1187          * submission can be a significant part of overall IO time.
1188          */
1189         if (workingset_read)
1190                 psi_memstall_enter(&pflags);
1191 
1192         ret = generic_make_request(bio);
1193 
1194         if (workingset_read)
1195                 psi_memstall_leave(&pflags);
1196 
1197         return ret;
1198 }
1199 EXPORT_SYMBOL(submit_bio);
1200 
1201 /**
1202  * blk_cloned_rq_check_limits - Helper function to check a cloned request
1203  *                              for new the queue limits
1204  * @q:  the queue
1205  * @rq: the request being checked
1206  *
1207  * Description:
1208  *    @rq may have been made based on weaker limitations of upper-level queues
1209  *    in request stacking drivers, and it may violate the limitation of @q.
1210  *    Since the block layer and the underlying device driver trust @rq
1211  *    after it is inserted to @q, it should be checked against @q before
1212  *    the insertion using this generic function.
1213  *
1214  *    Request stacking drivers like request-based dm may change the queue
1215  *    limits when retrying requests on other queues. Those requests need
1216  *    to be checked against the new queue limits again during dispatch.
1217  */
1218 static int blk_cloned_rq_check_limits(struct request_queue *q,
1219                                       struct request *rq)
1220 {
1221         if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
1222                 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
1223                         __func__, blk_rq_sectors(rq),
1224                         blk_queue_get_max_sectors(q, req_op(rq)));
1225                 return -EIO;
1226         }
1227 
1228         /*
1229          * queue's settings related to segment counting like q->bounce_pfn
1230          * may differ from that of other stacking queues.
1231          * Recalculate it to check the request correctly on this queue's
1232          * limitation.
1233          */
1234         rq->nr_phys_segments = blk_recalc_rq_segments(rq);
1235         if (rq->nr_phys_segments > queue_max_segments(q)) {
1236                 printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
1237                         __func__, rq->nr_phys_segments, queue_max_segments(q));
1238                 return -EIO;
1239         }
1240 
1241         return 0;
1242 }
1243 
1244 /**
1245  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1246  * @q:  the queue to submit the request
1247  * @rq: the request being queued
1248  */
1249 blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1250 {
1251         if (blk_cloned_rq_check_limits(q, rq))
1252                 return BLK_STS_IOERR;
1253 
1254         if (rq->rq_disk &&
1255             should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
1256                 return BLK_STS_IOERR;
1257 
1258         if (blk_queue_io_stat(q))
1259                 blk_account_io_start(rq, true);
1260 
1261         /*
1262          * Since we have a scheduler attached on the top device,
1263          * bypass a potential scheduler on the bottom device for
1264          * insert.
1265          */
1266         return blk_mq_request_issue_directly(rq, true);
1267 }
1268 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1269 
1270 /**
1271  * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1272  * @rq: request to examine
1273  *
1274  * Description:
1275  *     A request could be merge of IOs which require different failure
1276  *     handling.  This function determines the number of bytes which
1277  *     can be failed from the beginning of the request without
1278  *     crossing into area which need to be retried further.
1279  *
1280  * Return:
1281  *     The number of bytes to fail.
1282  */
1283 unsigned int blk_rq_err_bytes(const struct request *rq)
1284 {
1285         unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1286         unsigned int bytes = 0;
1287         struct bio *bio;
1288 
1289         if (!(rq->rq_flags & RQF_MIXED_MERGE))
1290                 return blk_rq_bytes(rq);
1291 
1292         /*
1293          * Currently the only 'mixing' which can happen is between
1294          * different fastfail types.  We can safely fail portions
1295          * which have all the failfast bits that the first one has -
1296          * the ones which are at least as eager to fail as the first
1297          * one.
1298          */
1299         for (bio = rq->bio; bio; bio = bio->bi_next) {
1300                 if ((bio->bi_opf & ff) != ff)
1301                         break;
1302                 bytes += bio->bi_iter.bi_size;
1303         }
1304 
1305         /* this could lead to infinite loop */
1306         BUG_ON(blk_rq_bytes(rq) && !bytes);
1307         return bytes;
1308 }
1309 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
1310 
1311 void blk_account_io_completion(struct request *req, unsigned int bytes)
1312 {
1313         if (req->part && blk_do_io_stat(req)) {
1314                 const int sgrp = op_stat_group(req_op(req));
1315                 struct hd_struct *part;
1316 
1317                 part_stat_lock();
1318                 part = req->part;
1319                 part_stat_add(part, sectors[sgrp], bytes >> 9);
1320                 part_stat_unlock();
1321         }
1322 }
1323 
1324 void blk_account_io_done(struct request *req, u64 now)
1325 {
1326         /*
1327          * Account IO completion.  flush_rq isn't accounted as a
1328          * normal IO on queueing nor completion.  Accounting the
1329          * containing request is enough.
1330          */
1331         if (req->part && blk_do_io_stat(req) &&
1332             !(req->rq_flags & RQF_FLUSH_SEQ)) {
1333                 const int sgrp = op_stat_group(req_op(req));
1334                 struct hd_struct *part;
1335 
1336                 part_stat_lock();
1337                 part = req->part;
1338 
1339                 update_io_ticks(part, jiffies);
1340                 part_stat_inc(part, ios[sgrp]);
1341                 part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);
1342                 part_stat_add(part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns));
1343                 part_dec_in_flight(req->q, part, rq_data_dir(req));
1344 
1345                 hd_struct_put(part);
1346                 part_stat_unlock();
1347         }
1348 }
1349 
1350 void blk_account_io_start(struct request *rq, bool new_io)
1351 {
1352         struct hd_struct *part;
1353         int rw = rq_data_dir(rq);
1354 
1355         if (!blk_do_io_stat(rq))
1356                 return;
1357 
1358         part_stat_lock();
1359 
1360         if (!new_io) {
1361                 part = rq->part;
1362                 part_stat_inc(part, merges[rw]);
1363         } else {
1364                 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
1365                 if (!hd_struct_try_get(part)) {
1366                         /*
1367                          * The partition is already being removed,
1368                          * the request will be accounted on the disk only
1369                          *
1370                          * We take a reference on disk->part0 although that
1371                          * partition will never be deleted, so we can treat
1372                          * it as any other partition.
1373                          */
1374                         part = &rq->rq_disk->part0;
1375                         hd_struct_get(part);
1376                 }
1377                 part_inc_in_flight(rq->q, part, rw);
1378                 rq->part = part;
1379         }
1380 
1381         update_io_ticks(part, jiffies);
1382 
1383         part_stat_unlock();
1384 }
1385 
1386 /*
1387  * Steal bios from a request and add them to a bio list.
1388  * The request must not have been partially completed before.
1389  */
1390 void blk_steal_bios(struct bio_list *list, struct request *rq)
1391 {
1392         if (rq->bio) {
1393                 if (list->tail)
1394                         list->tail->bi_next = rq->bio;
1395                 else
1396                         list->head = rq->bio;
1397                 list->tail = rq->biotail;
1398 
1399                 rq->bio = NULL;
1400                 rq->biotail = NULL;
1401         }
1402 
1403         rq->__data_len = 0;
1404 }
1405 EXPORT_SYMBOL_GPL(blk_steal_bios);
1406 
1407 /**
1408  * blk_update_request - Special helper function for request stacking drivers
1409  * @req:      the request being processed
1410  * @error:    block status code
1411  * @nr_bytes: number of bytes to complete @req
1412  *
1413  * Description:
1414  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
1415  *     the request structure even if @req doesn't have leftover.
1416  *     If @req has leftover, sets it up for the next range of segments.
1417  *
1418  *     This special helper function is only for request stacking drivers
1419  *     (e.g. request-based dm) so that they can handle partial completion.
1420  *     Actual device drivers should use blk_mq_end_request instead.
1421  *
1422  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
1423  *     %false return from this function.
1424  *
1425  * Note:
1426  *      The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
1427  *      blk_rq_bytes() and in blk_update_request().
1428  *
1429  * Return:
1430  *     %false - this request doesn't have any more data
1431  *     %true  - this request has more data
1432  **/
1433 bool blk_update_request(struct request *req, blk_status_t error,
1434                 unsigned int nr_bytes)
1435 {
1436         int total_bytes;
1437 
1438         trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
1439 
1440         if (!req->bio)
1441                 return false;
1442 
1443 #ifdef CONFIG_BLK_DEV_INTEGRITY
1444         if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
1445             error == BLK_STS_OK)
1446                 req->q->integrity.profile->complete_fn(req, nr_bytes);
1447 #endif
1448 
1449         if (unlikely(error && !blk_rq_is_passthrough(req) &&
1450                      !(req->rq_flags & RQF_QUIET)))
1451                 print_req_error(req, error, __func__);
1452 
1453         blk_account_io_completion(req, nr_bytes);
1454 
1455         total_bytes = 0;
1456         while (req->bio) {
1457                 struct bio *bio = req->bio;
1458                 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
1459 
1460                 if (bio_bytes == bio->bi_iter.bi_size)
1461                         req->bio = bio->bi_next;
1462 
1463                 /* Completion has already been traced */
1464                 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1465                 req_bio_endio(req, bio, bio_bytes, error);
1466 
1467                 total_bytes += bio_bytes;
1468                 nr_bytes -= bio_bytes;
1469 
1470                 if (!nr_bytes)
1471                         break;
1472         }
1473 
1474         /*
1475          * completely done
1476          */
1477         if (!req->bio) {
1478                 /*
1479                  * Reset counters so that the request stacking driver
1480                  * can find how many bytes remain in the request
1481                  * later.
1482                  */
1483                 req->__data_len = 0;
1484                 return false;
1485         }
1486 
1487         req->__data_len -= total_bytes;
1488 
1489         /* update sector only for requests with clear definition of sector */
1490         if (!blk_rq_is_passthrough(req))
1491                 req->__sector += total_bytes >> 9;
1492 
1493         /* mixed attributes always follow the first bio */
1494         if (req->rq_flags & RQF_MIXED_MERGE) {
1495                 req->cmd_flags &= ~REQ_FAILFAST_MASK;
1496                 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
1497         }
1498 
1499         if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
1500                 /*
1501                  * If total number of sectors is less than the first segment
1502                  * size, something has gone terribly wrong.
1503                  */
1504                 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
1505                         blk_dump_rq_flags(req, "request botched");
1506                         req->__data_len = blk_rq_cur_bytes(req);
1507                 }
1508 
1509                 /* recalculate the number of segments */
1510                 req->nr_phys_segments = blk_recalc_rq_segments(req);
1511         }
1512 
1513         return true;
1514 }
1515 EXPORT_SYMBOL_GPL(blk_update_request);
1516 
1517 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1518 /**
1519  * rq_flush_dcache_pages - Helper function to flush all pages in a request
1520  * @rq: the request to be flushed
1521  *
1522  * Description:
1523  *     Flush all pages in @rq.
1524  */
1525 void rq_flush_dcache_pages(struct request *rq)
1526 {
1527         struct req_iterator iter;
1528         struct bio_vec bvec;
1529 
1530         rq_for_each_segment(bvec, rq, iter)
1531                 flush_dcache_page(bvec.bv_page);
1532 }
1533 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
1534 #endif
1535 
1536 /**
1537  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
1538  * @q : the queue of the device being checked
1539  *
1540  * Description:
1541  *    Check if underlying low-level drivers of a device are busy.
1542  *    If the drivers want to export their busy state, they must set own
1543  *    exporting function using blk_queue_lld_busy() first.
1544  *
1545  *    Basically, this function is used only by request stacking drivers
1546  *    to stop dispatching requests to underlying devices when underlying
1547  *    devices are busy.  This behavior helps more I/O merging on the queue
1548  *    of the request stacking driver and prevents I/O throughput regression
1549  *    on burst I/O load.
1550  *
1551  * Return:
1552  *    0 - Not busy (The request stacking driver should dispatch request)
1553  *    1 - Busy (The request stacking driver should stop dispatching request)
1554  */
1555 int blk_lld_busy(struct request_queue *q)
1556 {
1557         if (queue_is_mq(q) && q->mq_ops->busy)
1558                 return q->mq_ops->busy(q);
1559 
1560         return 0;
1561 }
1562 EXPORT_SYMBOL_GPL(blk_lld_busy);
1563 
1564 /**
1565  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
1566  * @rq: the clone request to be cleaned up
1567  *
1568  * Description:
1569  *     Free all bios in @rq for a cloned request.
1570  */
1571 void blk_rq_unprep_clone(struct request *rq)
1572 {
1573         struct bio *bio;
1574 
1575         while ((bio = rq->bio) != NULL) {
1576                 rq->bio = bio->bi_next;
1577 
1578                 bio_put(bio);
1579         }
1580 }
1581 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
1582 
1583 /*
1584  * Copy attributes of the original request to the clone request.
1585  * The actual data parts (e.g. ->cmd, ->sense) are not copied.
1586  */
1587 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
1588 {
1589         dst->__sector = blk_rq_pos(src);
1590         dst->__data_len = blk_rq_bytes(src);
1591         if (src->rq_flags & RQF_SPECIAL_PAYLOAD) {
1592                 dst->rq_flags |= RQF_SPECIAL_PAYLOAD;
1593                 dst->special_vec = src->special_vec;
1594         }
1595         dst->nr_phys_segments = src->nr_phys_segments;
1596         dst->ioprio = src->ioprio;
1597         dst->extra_len = src->extra_len;
1598 }
1599 
1600 /**
1601  * blk_rq_prep_clone - Helper function to setup clone request
1602  * @rq: the request to be setup
1603  * @rq_src: original request to be cloned
1604  * @bs: bio_set that bios for clone are allocated from
1605  * @gfp_mask: memory allocation mask for bio
1606  * @bio_ctr: setup function to be called for each clone bio.
1607  *           Returns %0 for success, non %0 for failure.
1608  * @data: private data to be passed to @bio_ctr
1609  *
1610  * Description:
1611  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
1612  *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
1613  *     are not copied, and copying such parts is the caller's responsibility.
1614  *     Also, pages which the original bios are pointing to are not copied
1615  *     and the cloned bios just point same pages.
1616  *     So cloned bios must be completed before original bios, which means
1617  *     the caller must complete @rq before @rq_src.
1618  */
1619 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
1620                       struct bio_set *bs, gfp_t gfp_mask,
1621                       int (*bio_ctr)(struct bio *, struct bio *, void *),
1622                       void *data)
1623 {
1624         struct bio *bio, *bio_src;
1625 
1626         if (!bs)
1627                 bs = &fs_bio_set;
1628 
1629         __rq_for_each_bio(bio_src, rq_src) {
1630                 bio = bio_clone_fast(bio_src, gfp_mask, bs);
1631                 if (!bio)
1632                         goto free_and_out;
1633 
1634                 if (bio_ctr && bio_ctr(bio, bio_src, data))
1635                         goto free_and_out;
1636 
1637                 if (rq->bio) {
1638                         rq->biotail->bi_next = bio;
1639                         rq->biotail = bio;
1640                 } else
1641                         rq->bio = rq->biotail = bio;
1642         }
1643 
1644         __blk_rq_prep_clone(rq, rq_src);
1645 
1646         return 0;
1647 
1648 free_and_out:
1649         if (bio)
1650                 bio_put(bio);
1651         blk_rq_unprep_clone(rq);
1652 
1653         return -ENOMEM;
1654 }
1655 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
1656 
1657 int kblockd_schedule_work(struct work_struct *work)
1658 {
1659         return queue_work(kblockd_workqueue, work);
1660 }
1661 EXPORT_SYMBOL(kblockd_schedule_work);
1662 
1663 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
1664                                 unsigned long delay)
1665 {
1666         return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
1667 }
1668 EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
1669 
1670 /**
1671  * blk_start_plug - initialize blk_plug and track it inside the task_struct
1672  * @plug:       The &struct blk_plug that needs to be initialized
1673  *
1674  * Description:
1675  *   blk_start_plug() indicates to the block layer an intent by the caller
1676  *   to submit multiple I/O requests in a batch.  The block layer may use
1677  *   this hint to defer submitting I/Os from the caller until blk_finish_plug()
1678  *   is called.  However, the block layer may choose to submit requests
1679  *   before a call to blk_finish_plug() if the number of queued I/Os
1680  *   exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than
1681  *   %BLK_PLUG_FLUSH_SIZE.  The queued I/Os may also be submitted early if
1682  *   the task schedules (see below).
1683  *
1684  *   Tracking blk_plug inside the task_struct will help with auto-flushing the
1685  *   pending I/O should the task end up blocking between blk_start_plug() and
1686  *   blk_finish_plug(). This is important from a performance perspective, but
1687  *   also ensures that we don't deadlock. For instance, if the task is blocking
1688  *   for a memory allocation, memory reclaim could end up wanting to free a
1689  *   page belonging to that request that is currently residing in our private
1690  *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
1691  *   this kind of deadlock.
1692  */
1693 void blk_start_plug(struct blk_plug *plug)
1694 {
1695         struct task_struct *tsk = current;
1696 
1697         /*
1698          * If this is a nested plug, don't actually assign it.
1699          */
1700         if (tsk->plug)
1701                 return;
1702 
1703         INIT_LIST_HEAD(&plug->mq_list);
1704         INIT_LIST_HEAD(&plug->cb_list);
1705         plug->rq_count = 0;
1706         plug->multiple_queues = false;
1707 
1708         /*
1709          * Store ordering should not be needed here, since a potential
1710          * preempt will imply a full memory barrier
1711          */
1712         tsk->plug = plug;
1713 }
1714 EXPORT_SYMBOL(blk_start_plug);
1715 
1716 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
1717 {
1718         LIST_HEAD(callbacks);
1719 
1720         while (!list_empty(&plug->cb_list)) {
1721                 list_splice_init(&plug->cb_list, &callbacks);
1722 
1723                 while (!list_empty(&callbacks)) {
1724                         struct blk_plug_cb *cb = list_first_entry(&callbacks,
1725                                                           struct blk_plug_cb,
1726                                                           list);
1727                         list_del(&cb->list);
1728                         cb->callback(cb, from_schedule);
1729                 }
1730         }
1731 }
1732 
1733 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
1734                                       int size)
1735 {
1736         struct blk_plug *plug = current->plug;
1737         struct blk_plug_cb *cb;
1738 
1739         if (!plug)
1740                 return NULL;
1741 
1742         list_for_each_entry(cb, &plug->cb_list, list)
1743                 if (cb->callback == unplug && cb->data == data)
1744                         return cb;
1745 
1746         /* Not currently on the callback list */
1747         BUG_ON(size < sizeof(*cb));
1748         cb = kzalloc(size, GFP_ATOMIC);
1749         if (cb) {
1750                 cb->data = data;
1751                 cb->callback = unplug;
1752                 list_add(&cb->list, &plug->cb_list);
1753         }
1754         return cb;
1755 }
1756 EXPORT_SYMBOL(blk_check_plugged);
1757 
1758 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1759 {
1760         flush_plug_callbacks(plug, from_schedule);
1761 
1762         if (!list_empty(&plug->mq_list))
1763                 blk_mq_flush_plug_list(plug, from_schedule);
1764 }
1765 
1766 /**
1767  * blk_finish_plug - mark the end of a batch of submitted I/O
1768  * @plug:       The &struct blk_plug passed to blk_start_plug()
1769  *
1770  * Description:
1771  * Indicate that a batch of I/O submissions is complete.  This function
1772  * must be paired with an initial call to blk_start_plug().  The intent
1773  * is to allow the block layer to optimize I/O submission.  See the
1774  * documentation for blk_start_plug() for more information.
1775  */
1776 void blk_finish_plug(struct blk_plug *plug)
1777 {
1778         if (plug != current->plug)
1779                 return;
1780         blk_flush_plug_list(plug, false);
1781 
1782         current->plug = NULL;
1783 }
1784 EXPORT_SYMBOL(blk_finish_plug);
1785 
1786 int __init blk_dev_init(void)
1787 {
1788         BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
1789         BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1790                         sizeof_field(struct request, cmd_flags));
1791         BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1792                         sizeof_field(struct bio, bi_opf));
1793 
1794         /* used for unplugging and affects IO latency/throughput - HIGHPRI */
1795         kblockd_workqueue = alloc_workqueue("kblockd",
1796                                             WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1797         if (!kblockd_workqueue)
1798                 panic("Failed to create kblockd\n");
1799 
1800         blk_requestq_cachep = kmem_cache_create("request_queue",
1801                         sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
1802 
1803 #ifdef CONFIG_DEBUG_FS
1804         blk_debugfs_root = debugfs_create_dir("block", NULL);
1805 #endif
1806 
1807         return 0;
1808 }
1809 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp