~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/block/blk-core.c

Version: ~ [ linux-5.15-rc5 ] ~ [ linux-5.14.11 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.72 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.152 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.210 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.250 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.286 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.288 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Copyright (C) 1991, 1992 Linus Torvalds
  4  * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
  5  * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
  6  * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
  7  * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
  8  *      -  July2000
  9  * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 10  */
 11 
 12 /*
 13  * This handles all read/write requests to block devices
 14  */
 15 #include <linux/kernel.h>
 16 #include <linux/module.h>
 17 #include <linux/backing-dev.h>
 18 #include <linux/bio.h>
 19 #include <linux/blkdev.h>
 20 #include <linux/blk-mq.h>
 21 #include <linux/highmem.h>
 22 #include <linux/mm.h>
 23 #include <linux/kernel_stat.h>
 24 #include <linux/string.h>
 25 #include <linux/init.h>
 26 #include <linux/completion.h>
 27 #include <linux/slab.h>
 28 #include <linux/swap.h>
 29 #include <linux/writeback.h>
 30 #include <linux/task_io_accounting_ops.h>
 31 #include <linux/fault-inject.h>
 32 #include <linux/list_sort.h>
 33 #include <linux/delay.h>
 34 #include <linux/ratelimit.h>
 35 #include <linux/pm_runtime.h>
 36 #include <linux/blk-cgroup.h>
 37 #include <linux/debugfs.h>
 38 #include <linux/bpf.h>
 39 
 40 #define CREATE_TRACE_POINTS
 41 #include <trace/events/block.h>
 42 
 43 #include "blk.h"
 44 #include "blk-mq.h"
 45 #include "blk-mq-sched.h"
 46 #include "blk-pm.h"
 47 #include "blk-rq-qos.h"
 48 
 49 #ifdef CONFIG_DEBUG_FS
 50 struct dentry *blk_debugfs_root;
 51 #endif
 52 
 53 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
 54 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
 55 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
 56 EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
 57 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
 58 
 59 DEFINE_IDA(blk_queue_ida);
 60 
 61 /*
 62  * For queue allocation
 63  */
 64 struct kmem_cache *blk_requestq_cachep;
 65 
 66 /*
 67  * Controlling structure to kblockd
 68  */
 69 static struct workqueue_struct *kblockd_workqueue;
 70 
 71 /**
 72  * blk_queue_flag_set - atomically set a queue flag
 73  * @flag: flag to be set
 74  * @q: request queue
 75  */
 76 void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
 77 {
 78         set_bit(flag, &q->queue_flags);
 79 }
 80 EXPORT_SYMBOL(blk_queue_flag_set);
 81 
 82 /**
 83  * blk_queue_flag_clear - atomically clear a queue flag
 84  * @flag: flag to be cleared
 85  * @q: request queue
 86  */
 87 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
 88 {
 89         clear_bit(flag, &q->queue_flags);
 90 }
 91 EXPORT_SYMBOL(blk_queue_flag_clear);
 92 
 93 /**
 94  * blk_queue_flag_test_and_set - atomically test and set a queue flag
 95  * @flag: flag to be set
 96  * @q: request queue
 97  *
 98  * Returns the previous value of @flag - 0 if the flag was not set and 1 if
 99  * the flag was already set.
100  */
101 bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
102 {
103         return test_and_set_bit(flag, &q->queue_flags);
104 }
105 EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
106 
107 void blk_rq_init(struct request_queue *q, struct request *rq)
108 {
109         memset(rq, 0, sizeof(*rq));
110 
111         INIT_LIST_HEAD(&rq->queuelist);
112         rq->q = q;
113         rq->__sector = (sector_t) -1;
114         INIT_HLIST_NODE(&rq->hash);
115         RB_CLEAR_NODE(&rq->rb_node);
116         rq->tag = -1;
117         rq->internal_tag = -1;
118         rq->start_time_ns = ktime_get_ns();
119         rq->part = NULL;
120         refcount_set(&rq->ref, 1);
121 }
122 EXPORT_SYMBOL(blk_rq_init);
123 
124 static const struct {
125         int             errno;
126         const char      *name;
127 } blk_errors[] = {
128         [BLK_STS_OK]            = { 0,          "" },
129         [BLK_STS_NOTSUPP]       = { -EOPNOTSUPP, "operation not supported" },
130         [BLK_STS_TIMEOUT]       = { -ETIMEDOUT, "timeout" },
131         [BLK_STS_NOSPC]         = { -ENOSPC,    "critical space allocation" },
132         [BLK_STS_TRANSPORT]     = { -ENOLINK,   "recoverable transport" },
133         [BLK_STS_TARGET]        = { -EREMOTEIO, "critical target" },
134         [BLK_STS_NEXUS]         = { -EBADE,     "critical nexus" },
135         [BLK_STS_MEDIUM]        = { -ENODATA,   "critical medium" },
136         [BLK_STS_PROTECTION]    = { -EILSEQ,    "protection" },
137         [BLK_STS_RESOURCE]      = { -ENOMEM,    "kernel resource" },
138         [BLK_STS_DEV_RESOURCE]  = { -EBUSY,     "device resource" },
139         [BLK_STS_AGAIN]         = { -EAGAIN,    "nonblocking retry" },
140 
141         /* device mapper special case, should not leak out: */
142         [BLK_STS_DM_REQUEUE]    = { -EREMCHG, "dm internal retry" },
143 
144         /* everything else not covered above: */
145         [BLK_STS_IOERR]         = { -EIO,       "I/O" },
146 };
147 
148 blk_status_t errno_to_blk_status(int errno)
149 {
150         int i;
151 
152         for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
153                 if (blk_errors[i].errno == errno)
154                         return (__force blk_status_t)i;
155         }
156 
157         return BLK_STS_IOERR;
158 }
159 EXPORT_SYMBOL_GPL(errno_to_blk_status);
160 
161 int blk_status_to_errno(blk_status_t status)
162 {
163         int idx = (__force int)status;
164 
165         if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
166                 return -EIO;
167         return blk_errors[idx].errno;
168 }
169 EXPORT_SYMBOL_GPL(blk_status_to_errno);
170 
171 static void print_req_error(struct request *req, blk_status_t status)
172 {
173         int idx = (__force int)status;
174 
175         if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
176                 return;
177 
178         printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu flags %x\n",
179                                 __func__, blk_errors[idx].name,
180                                 req->rq_disk ?  req->rq_disk->disk_name : "?",
181                                 (unsigned long long)blk_rq_pos(req),
182                                 req->cmd_flags);
183 }
184 
185 static void req_bio_endio(struct request *rq, struct bio *bio,
186                           unsigned int nbytes, blk_status_t error)
187 {
188         if (error)
189                 bio->bi_status = error;
190 
191         if (unlikely(rq->rq_flags & RQF_QUIET))
192                 bio_set_flag(bio, BIO_QUIET);
193 
194         bio_advance(bio, nbytes);
195 
196         /* don't actually finish bio if it's part of flush sequence */
197         if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
198                 bio_endio(bio);
199 }
200 
201 void blk_dump_rq_flags(struct request *rq, char *msg)
202 {
203         printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
204                 rq->rq_disk ? rq->rq_disk->disk_name : "?",
205                 (unsigned long long) rq->cmd_flags);
206 
207         printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
208                (unsigned long long)blk_rq_pos(rq),
209                blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
210         printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
211                rq->bio, rq->biotail, blk_rq_bytes(rq));
212 }
213 EXPORT_SYMBOL(blk_dump_rq_flags);
214 
215 /**
216  * blk_sync_queue - cancel any pending callbacks on a queue
217  * @q: the queue
218  *
219  * Description:
220  *     The block layer may perform asynchronous callback activity
221  *     on a queue, such as calling the unplug function after a timeout.
222  *     A block device may call blk_sync_queue to ensure that any
223  *     such activity is cancelled, thus allowing it to release resources
224  *     that the callbacks might use. The caller must already have made sure
225  *     that its ->make_request_fn will not re-add plugging prior to calling
226  *     this function.
227  *
228  *     This function does not cancel any asynchronous activity arising
229  *     out of elevator or throttling code. That would require elevator_exit()
230  *     and blkcg_exit_queue() to be called with queue lock initialized.
231  *
232  */
233 void blk_sync_queue(struct request_queue *q)
234 {
235         del_timer_sync(&q->timeout);
236         cancel_work_sync(&q->timeout_work);
237 }
238 EXPORT_SYMBOL(blk_sync_queue);
239 
240 /**
241  * blk_set_pm_only - increment pm_only counter
242  * @q: request queue pointer
243  */
244 void blk_set_pm_only(struct request_queue *q)
245 {
246         atomic_inc(&q->pm_only);
247 }
248 EXPORT_SYMBOL_GPL(blk_set_pm_only);
249 
250 void blk_clear_pm_only(struct request_queue *q)
251 {
252         int pm_only;
253 
254         pm_only = atomic_dec_return(&q->pm_only);
255         WARN_ON_ONCE(pm_only < 0);
256         if (pm_only == 0)
257                 wake_up_all(&q->mq_freeze_wq);
258 }
259 EXPORT_SYMBOL_GPL(blk_clear_pm_only);
260 
261 void blk_put_queue(struct request_queue *q)
262 {
263         kobject_put(&q->kobj);
264 }
265 EXPORT_SYMBOL(blk_put_queue);
266 
267 void blk_set_queue_dying(struct request_queue *q)
268 {
269         blk_queue_flag_set(QUEUE_FLAG_DYING, q);
270 
271         /*
272          * When queue DYING flag is set, we need to block new req
273          * entering queue, so we call blk_freeze_queue_start() to
274          * prevent I/O from crossing blk_queue_enter().
275          */
276         blk_freeze_queue_start(q);
277 
278         if (queue_is_mq(q))
279                 blk_mq_wake_waiters(q);
280 
281         /* Make blk_queue_enter() reexamine the DYING flag. */
282         wake_up_all(&q->mq_freeze_wq);
283 }
284 EXPORT_SYMBOL_GPL(blk_set_queue_dying);
285 
286 /**
287  * blk_cleanup_queue - shutdown a request queue
288  * @q: request queue to shutdown
289  *
290  * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
291  * put it.  All future requests will be failed immediately with -ENODEV.
292  */
293 void blk_cleanup_queue(struct request_queue *q)
294 {
295         /* mark @q DYING, no new request or merges will be allowed afterwards */
296         mutex_lock(&q->sysfs_lock);
297         blk_set_queue_dying(q);
298 
299         blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
300         blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
301         blk_queue_flag_set(QUEUE_FLAG_DYING, q);
302         mutex_unlock(&q->sysfs_lock);
303 
304         /*
305          * Drain all requests queued before DYING marking. Set DEAD flag to
306          * prevent that q->request_fn() gets invoked after draining finished.
307          */
308         blk_freeze_queue(q);
309 
310         rq_qos_exit(q);
311 
312         blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
313 
314         /* for synchronous bio-based driver finish in-flight integrity i/o */
315         blk_flush_integrity();
316 
317         /* @q won't process any more request, flush async actions */
318         del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
319         blk_sync_queue(q);
320 
321         if (queue_is_mq(q))
322                 blk_mq_exit_queue(q);
323 
324         /*
325          * In theory, request pool of sched_tags belongs to request queue.
326          * However, the current implementation requires tag_set for freeing
327          * requests, so free the pool now.
328          *
329          * Queue has become frozen, there can't be any in-queue requests, so
330          * it is safe to free requests now.
331          */
332         mutex_lock(&q->sysfs_lock);
333         if (q->elevator)
334                 blk_mq_sched_free_requests(q);
335         mutex_unlock(&q->sysfs_lock);
336 
337         percpu_ref_exit(&q->q_usage_counter);
338 
339         /* @q is and will stay empty, shutdown and put */
340         blk_put_queue(q);
341 }
342 EXPORT_SYMBOL(blk_cleanup_queue);
343 
344 struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
345 {
346         return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
347 }
348 EXPORT_SYMBOL(blk_alloc_queue);
349 
350 /**
351  * blk_queue_enter() - try to increase q->q_usage_counter
352  * @q: request queue pointer
353  * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT
354  */
355 int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
356 {
357         const bool pm = flags & BLK_MQ_REQ_PREEMPT;
358 
359         while (true) {
360                 bool success = false;
361 
362                 rcu_read_lock();
363                 if (percpu_ref_tryget_live(&q->q_usage_counter)) {
364                         /*
365                          * The code that increments the pm_only counter is
366                          * responsible for ensuring that that counter is
367                          * globally visible before the queue is unfrozen.
368                          */
369                         if (pm || !blk_queue_pm_only(q)) {
370                                 success = true;
371                         } else {
372                                 percpu_ref_put(&q->q_usage_counter);
373                         }
374                 }
375                 rcu_read_unlock();
376 
377                 if (success)
378                         return 0;
379 
380                 if (flags & BLK_MQ_REQ_NOWAIT)
381                         return -EBUSY;
382 
383                 /*
384                  * read pair of barrier in blk_freeze_queue_start(),
385                  * we need to order reading __PERCPU_REF_DEAD flag of
386                  * .q_usage_counter and reading .mq_freeze_depth or
387                  * queue dying flag, otherwise the following wait may
388                  * never return if the two reads are reordered.
389                  */
390                 smp_rmb();
391 
392                 wait_event(q->mq_freeze_wq,
393                            (!q->mq_freeze_depth &&
394                             (pm || (blk_pm_request_resume(q),
395                                     !blk_queue_pm_only(q)))) ||
396                            blk_queue_dying(q));
397                 if (blk_queue_dying(q))
398                         return -ENODEV;
399         }
400 }
401 
402 void blk_queue_exit(struct request_queue *q)
403 {
404         percpu_ref_put(&q->q_usage_counter);
405 }
406 
407 static void blk_queue_usage_counter_release(struct percpu_ref *ref)
408 {
409         struct request_queue *q =
410                 container_of(ref, struct request_queue, q_usage_counter);
411 
412         wake_up_all(&q->mq_freeze_wq);
413 }
414 
415 static void blk_rq_timed_out_timer(struct timer_list *t)
416 {
417         struct request_queue *q = from_timer(q, t, timeout);
418 
419         kblockd_schedule_work(&q->timeout_work);
420 }
421 
422 static void blk_timeout_work(struct work_struct *work)
423 {
424 }
425 
426 /**
427  * blk_alloc_queue_node - allocate a request queue
428  * @gfp_mask: memory allocation flags
429  * @node_id: NUMA node to allocate memory from
430  */
431 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
432 {
433         struct request_queue *q;
434         int ret;
435 
436         q = kmem_cache_alloc_node(blk_requestq_cachep,
437                                 gfp_mask | __GFP_ZERO, node_id);
438         if (!q)
439                 return NULL;
440 
441         INIT_LIST_HEAD(&q->queue_head);
442         q->last_merge = NULL;
443 
444         q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
445         if (q->id < 0)
446                 goto fail_q;
447 
448         ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
449         if (ret)
450                 goto fail_id;
451 
452         q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
453         if (!q->backing_dev_info)
454                 goto fail_split;
455 
456         q->stats = blk_alloc_queue_stats();
457         if (!q->stats)
458                 goto fail_stats;
459 
460         q->backing_dev_info->ra_pages = VM_READAHEAD_PAGES;
461         q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
462         q->backing_dev_info->name = "block";
463         q->node = node_id;
464 
465         timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
466                     laptop_mode_timer_fn, 0);
467         timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
468         INIT_WORK(&q->timeout_work, blk_timeout_work);
469         INIT_LIST_HEAD(&q->icq_list);
470 #ifdef CONFIG_BLK_CGROUP
471         INIT_LIST_HEAD(&q->blkg_list);
472 #endif
473 
474         kobject_init(&q->kobj, &blk_queue_ktype);
475 
476 #ifdef CONFIG_BLK_DEV_IO_TRACE
477         mutex_init(&q->blk_trace_mutex);
478 #endif
479         mutex_init(&q->sysfs_lock);
480         spin_lock_init(&q->queue_lock);
481 
482         init_waitqueue_head(&q->mq_freeze_wq);
483         mutex_init(&q->mq_freeze_lock);
484 
485         /*
486          * Init percpu_ref in atomic mode so that it's faster to shutdown.
487          * See blk_register_queue() for details.
488          */
489         if (percpu_ref_init(&q->q_usage_counter,
490                                 blk_queue_usage_counter_release,
491                                 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
492                 goto fail_bdi;
493 
494         if (blkcg_init_queue(q))
495                 goto fail_ref;
496 
497         return q;
498 
499 fail_ref:
500         percpu_ref_exit(&q->q_usage_counter);
501 fail_bdi:
502         blk_free_queue_stats(q->stats);
503 fail_stats:
504         bdi_put(q->backing_dev_info);
505 fail_split:
506         bioset_exit(&q->bio_split);
507 fail_id:
508         ida_simple_remove(&blk_queue_ida, q->id);
509 fail_q:
510         kmem_cache_free(blk_requestq_cachep, q);
511         return NULL;
512 }
513 EXPORT_SYMBOL(blk_alloc_queue_node);
514 
515 bool blk_get_queue(struct request_queue *q)
516 {
517         if (likely(!blk_queue_dying(q))) {
518                 __blk_get_queue(q);
519                 return true;
520         }
521 
522         return false;
523 }
524 EXPORT_SYMBOL(blk_get_queue);
525 
526 /**
527  * blk_get_request - allocate a request
528  * @q: request queue to allocate a request for
529  * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
530  * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
531  */
532 struct request *blk_get_request(struct request_queue *q, unsigned int op,
533                                 blk_mq_req_flags_t flags)
534 {
535         struct request *req;
536 
537         WARN_ON_ONCE(op & REQ_NOWAIT);
538         WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
539 
540         req = blk_mq_alloc_request(q, op, flags);
541         if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
542                 q->mq_ops->initialize_rq_fn(req);
543 
544         return req;
545 }
546 EXPORT_SYMBOL(blk_get_request);
547 
548 void blk_put_request(struct request *req)
549 {
550         blk_mq_free_request(req);
551 }
552 EXPORT_SYMBOL(blk_put_request);
553 
554 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
555                             struct bio *bio)
556 {
557         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
558 
559         if (!ll_back_merge_fn(q, req, bio))
560                 return false;
561 
562         trace_block_bio_backmerge(q, req, bio);
563 
564         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
565                 blk_rq_set_mixed_merge(req);
566 
567         req->biotail->bi_next = bio;
568         req->biotail = bio;
569         req->__data_len += bio->bi_iter.bi_size;
570 
571         blk_account_io_start(req, false);
572         return true;
573 }
574 
575 bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
576                              struct bio *bio)
577 {
578         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
579 
580         if (!ll_front_merge_fn(q, req, bio))
581                 return false;
582 
583         trace_block_bio_frontmerge(q, req, bio);
584 
585         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
586                 blk_rq_set_mixed_merge(req);
587 
588         bio->bi_next = req->bio;
589         req->bio = bio;
590 
591         req->__sector = bio->bi_iter.bi_sector;
592         req->__data_len += bio->bi_iter.bi_size;
593 
594         blk_account_io_start(req, false);
595         return true;
596 }
597 
598 bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
599                 struct bio *bio)
600 {
601         unsigned short segments = blk_rq_nr_discard_segments(req);
602 
603         if (segments >= queue_max_discard_segments(q))
604                 goto no_merge;
605         if (blk_rq_sectors(req) + bio_sectors(bio) >
606             blk_rq_get_max_sectors(req, blk_rq_pos(req)))
607                 goto no_merge;
608 
609         req->biotail->bi_next = bio;
610         req->biotail = bio;
611         req->__data_len += bio->bi_iter.bi_size;
612         req->nr_phys_segments = segments + 1;
613 
614         blk_account_io_start(req, false);
615         return true;
616 no_merge:
617         req_set_nomerge(q, req);
618         return false;
619 }
620 
621 /**
622  * blk_attempt_plug_merge - try to merge with %current's plugged list
623  * @q: request_queue new bio is being queued at
624  * @bio: new bio being queued
625  * @same_queue_rq: pointer to &struct request that gets filled in when
626  * another request associated with @q is found on the plug list
627  * (optional, may be %NULL)
628  *
629  * Determine whether @bio being queued on @q can be merged with a request
630  * on %current's plugged list.  Returns %true if merge was successful,
631  * otherwise %false.
632  *
633  * Plugging coalesces IOs from the same issuer for the same purpose without
634  * going through @q->queue_lock.  As such it's more of an issuing mechanism
635  * than scheduling, and the request, while may have elvpriv data, is not
636  * added on the elevator at this point.  In addition, we don't have
637  * reliable access to the elevator outside queue lock.  Only check basic
638  * merging parameters without querying the elevator.
639  *
640  * Caller must ensure !blk_queue_nomerges(q) beforehand.
641  */
642 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
643                             struct request **same_queue_rq)
644 {
645         struct blk_plug *plug;
646         struct request *rq;
647         struct list_head *plug_list;
648 
649         plug = current->plug;
650         if (!plug)
651                 return false;
652 
653         plug_list = &plug->mq_list;
654 
655         list_for_each_entry_reverse(rq, plug_list, queuelist) {
656                 bool merged = false;
657 
658                 if (rq->q == q && same_queue_rq) {
659                         /*
660                          * Only blk-mq multiple hardware queues case checks the
661                          * rq in the same queue, there should be only one such
662                          * rq in a queue
663                          **/
664                         *same_queue_rq = rq;
665                 }
666 
667                 if (rq->q != q || !blk_rq_merge_ok(rq, bio))
668                         continue;
669 
670                 switch (blk_try_merge(rq, bio)) {
671                 case ELEVATOR_BACK_MERGE:
672                         merged = bio_attempt_back_merge(q, rq, bio);
673                         break;
674                 case ELEVATOR_FRONT_MERGE:
675                         merged = bio_attempt_front_merge(q, rq, bio);
676                         break;
677                 case ELEVATOR_DISCARD_MERGE:
678                         merged = bio_attempt_discard_merge(q, rq, bio);
679                         break;
680                 default:
681                         break;
682                 }
683 
684                 if (merged)
685                         return true;
686         }
687 
688         return false;
689 }
690 
691 void blk_init_request_from_bio(struct request *req, struct bio *bio)
692 {
693         if (bio->bi_opf & REQ_RAHEAD)
694                 req->cmd_flags |= REQ_FAILFAST_MASK;
695 
696         req->__sector = bio->bi_iter.bi_sector;
697         req->ioprio = bio_prio(bio);
698         req->write_hint = bio->bi_write_hint;
699         blk_rq_bio_prep(req->q, req, bio);
700 }
701 EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
702 
703 static void handle_bad_sector(struct bio *bio, sector_t maxsector)
704 {
705         char b[BDEVNAME_SIZE];
706 
707         printk(KERN_INFO "attempt to access beyond end of device\n");
708         printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
709                         bio_devname(bio, b), bio->bi_opf,
710                         (unsigned long long)bio_end_sector(bio),
711                         (long long)maxsector);
712 }
713 
714 #ifdef CONFIG_FAIL_MAKE_REQUEST
715 
716 static DECLARE_FAULT_ATTR(fail_make_request);
717 
718 static int __init setup_fail_make_request(char *str)
719 {
720         return setup_fault_attr(&fail_make_request, str);
721 }
722 __setup("fail_make_request=", setup_fail_make_request);
723 
724 static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
725 {
726         return part->make_it_fail && should_fail(&fail_make_request, bytes);
727 }
728 
729 static int __init fail_make_request_debugfs(void)
730 {
731         struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
732                                                 NULL, &fail_make_request);
733 
734         return PTR_ERR_OR_ZERO(dir);
735 }
736 
737 late_initcall(fail_make_request_debugfs);
738 
739 #else /* CONFIG_FAIL_MAKE_REQUEST */
740 
741 static inline bool should_fail_request(struct hd_struct *part,
742                                         unsigned int bytes)
743 {
744         return false;
745 }
746 
747 #endif /* CONFIG_FAIL_MAKE_REQUEST */
748 
749 static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
750 {
751         const int op = bio_op(bio);
752 
753         if (part->policy && op_is_write(op)) {
754                 char b[BDEVNAME_SIZE];
755 
756                 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
757                         return false;
758 
759                 WARN_ONCE(1,
760                        "generic_make_request: Trying to write "
761                         "to read-only block-device %s (partno %d)\n",
762                         bio_devname(bio, b), part->partno);
763                 /* Older lvm-tools actually trigger this */
764                 return false;
765         }
766 
767         return false;
768 }
769 
770 static noinline int should_fail_bio(struct bio *bio)
771 {
772         if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
773                 return -EIO;
774         return 0;
775 }
776 ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
777 
778 /*
779  * Check whether this bio extends beyond the end of the device or partition.
780  * This may well happen - the kernel calls bread() without checking the size of
781  * the device, e.g., when mounting a file system.
782  */
783 static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
784 {
785         unsigned int nr_sectors = bio_sectors(bio);
786 
787         if (nr_sectors && maxsector &&
788             (nr_sectors > maxsector ||
789              bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
790                 handle_bad_sector(bio, maxsector);
791                 return -EIO;
792         }
793         return 0;
794 }
795 
796 /*
797  * Remap block n of partition p to block n+start(p) of the disk.
798  */
799 static inline int blk_partition_remap(struct bio *bio)
800 {
801         struct hd_struct *p;
802         int ret = -EIO;
803 
804         rcu_read_lock();
805         p = __disk_get_part(bio->bi_disk, bio->bi_partno);
806         if (unlikely(!p))
807                 goto out;
808         if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
809                 goto out;
810         if (unlikely(bio_check_ro(bio, p)))
811                 goto out;
812 
813         /*
814          * Zone reset does not include bi_size so bio_sectors() is always 0.
815          * Include a test for the reset op code and perform the remap if needed.
816          */
817         if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) {
818                 if (bio_check_eod(bio, part_nr_sects_read(p)))
819                         goto out;
820                 bio->bi_iter.bi_sector += p->start_sect;
821                 trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
822                                       bio->bi_iter.bi_sector - p->start_sect);
823         }
824         bio->bi_partno = 0;
825         ret = 0;
826 out:
827         rcu_read_unlock();
828         return ret;
829 }
830 
831 static noinline_for_stack bool
832 generic_make_request_checks(struct bio *bio)
833 {
834         struct request_queue *q;
835         int nr_sectors = bio_sectors(bio);
836         blk_status_t status = BLK_STS_IOERR;
837         char b[BDEVNAME_SIZE];
838 
839         might_sleep();
840 
841         q = bio->bi_disk->queue;
842         if (unlikely(!q)) {
843                 printk(KERN_ERR
844                        "generic_make_request: Trying to access "
845                         "nonexistent block-device %s (%Lu)\n",
846                         bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
847                 goto end_io;
848         }
849 
850         /*
851          * For a REQ_NOWAIT based request, return -EOPNOTSUPP
852          * if queue is not a request based queue.
853          */
854         if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q))
855                 goto not_supported;
856 
857         if (should_fail_bio(bio))
858                 goto end_io;
859 
860         if (bio->bi_partno) {
861                 if (unlikely(blk_partition_remap(bio)))
862                         goto end_io;
863         } else {
864                 if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
865                         goto end_io;
866                 if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
867                         goto end_io;
868         }
869 
870         /*
871          * Filter flush bio's early so that make_request based
872          * drivers without flush support don't have to worry
873          * about them.
874          */
875         if (op_is_flush(bio->bi_opf) &&
876             !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
877                 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
878                 if (!nr_sectors) {
879                         status = BLK_STS_OK;
880                         goto end_io;
881                 }
882         }
883 
884         if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
885                 bio->bi_opf &= ~REQ_HIPRI;
886 
887         switch (bio_op(bio)) {
888         case REQ_OP_DISCARD:
889                 if (!blk_queue_discard(q))
890                         goto not_supported;
891                 break;
892         case REQ_OP_SECURE_ERASE:
893                 if (!blk_queue_secure_erase(q))
894                         goto not_supported;
895                 break;
896         case REQ_OP_WRITE_SAME:
897                 if (!q->limits.max_write_same_sectors)
898                         goto not_supported;
899                 break;
900         case REQ_OP_ZONE_RESET:
901                 if (!blk_queue_is_zoned(q))
902                         goto not_supported;
903                 break;
904         case REQ_OP_WRITE_ZEROES:
905                 if (!q->limits.max_write_zeroes_sectors)
906                         goto not_supported;
907                 break;
908         default:
909                 break;
910         }
911 
912         /*
913          * Various block parts want %current->io_context and lazy ioc
914          * allocation ends up trading a lot of pain for a small amount of
915          * memory.  Just allocate it upfront.  This may fail and block
916          * layer knows how to live with it.
917          */
918         create_io_context(GFP_ATOMIC, q->node);
919 
920         if (!blkcg_bio_issue_check(q, bio))
921                 return false;
922 
923         if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
924                 trace_block_bio_queue(q, bio);
925                 /* Now that enqueuing has been traced, we need to trace
926                  * completion as well.
927                  */
928                 bio_set_flag(bio, BIO_TRACE_COMPLETION);
929         }
930         return true;
931 
932 not_supported:
933         status = BLK_STS_NOTSUPP;
934 end_io:
935         bio->bi_status = status;
936         bio_endio(bio);
937         return false;
938 }
939 
940 /**
941  * generic_make_request - hand a buffer to its device driver for I/O
942  * @bio:  The bio describing the location in memory and on the device.
943  *
944  * generic_make_request() is used to make I/O requests of block
945  * devices. It is passed a &struct bio, which describes the I/O that needs
946  * to be done.
947  *
948  * generic_make_request() does not return any status.  The
949  * success/failure status of the request, along with notification of
950  * completion, is delivered asynchronously through the bio->bi_end_io
951  * function described (one day) else where.
952  *
953  * The caller of generic_make_request must make sure that bi_io_vec
954  * are set to describe the memory buffer, and that bi_dev and bi_sector are
955  * set to describe the device address, and the
956  * bi_end_io and optionally bi_private are set to describe how
957  * completion notification should be signaled.
958  *
959  * generic_make_request and the drivers it calls may use bi_next if this
960  * bio happens to be merged with someone else, and may resubmit the bio to
961  * a lower device by calling into generic_make_request recursively, which
962  * means the bio should NOT be touched after the call to ->make_request_fn.
963  */
964 blk_qc_t generic_make_request(struct bio *bio)
965 {
966         /*
967          * bio_list_on_stack[0] contains bios submitted by the current
968          * make_request_fn.
969          * bio_list_on_stack[1] contains bios that were submitted before
970          * the current make_request_fn, but that haven't been processed
971          * yet.
972          */
973         struct bio_list bio_list_on_stack[2];
974         blk_qc_t ret = BLK_QC_T_NONE;
975 
976         if (!generic_make_request_checks(bio))
977                 goto out;
978 
979         /*
980          * We only want one ->make_request_fn to be active at a time, else
981          * stack usage with stacked devices could be a problem.  So use
982          * current->bio_list to keep a list of requests submited by a
983          * make_request_fn function.  current->bio_list is also used as a
984          * flag to say if generic_make_request is currently active in this
985          * task or not.  If it is NULL, then no make_request is active.  If
986          * it is non-NULL, then a make_request is active, and new requests
987          * should be added at the tail
988          */
989         if (current->bio_list) {
990                 bio_list_add(&current->bio_list[0], bio);
991                 goto out;
992         }
993 
994         /* following loop may be a bit non-obvious, and so deserves some
995          * explanation.
996          * Before entering the loop, bio->bi_next is NULL (as all callers
997          * ensure that) so we have a list with a single bio.
998          * We pretend that we have just taken it off a longer list, so
999          * we assign bio_list to a pointer to the bio_list_on_stack,
1000          * thus initialising the bio_list of new bios to be
1001          * added.  ->make_request() may indeed add some more bios
1002          * through a recursive call to generic_make_request.  If it
1003          * did, we find a non-NULL value in bio_list and re-enter the loop
1004          * from the top.  In this case we really did just take the bio
1005          * of the top of the list (no pretending) and so remove it from
1006          * bio_list, and call into ->make_request() again.
1007          */
1008         BUG_ON(bio->bi_next);
1009         bio_list_init(&bio_list_on_stack[0]);
1010         current->bio_list = bio_list_on_stack;
1011         do {
1012                 struct request_queue *q = bio->bi_disk->queue;
1013                 blk_mq_req_flags_t flags = bio->bi_opf & REQ_NOWAIT ?
1014                         BLK_MQ_REQ_NOWAIT : 0;
1015 
1016                 if (likely(blk_queue_enter(q, flags) == 0)) {
1017                         struct bio_list lower, same;
1018 
1019                         /* Create a fresh bio_list for all subordinate requests */
1020                         bio_list_on_stack[1] = bio_list_on_stack[0];
1021                         bio_list_init(&bio_list_on_stack[0]);
1022                         ret = q->make_request_fn(q, bio);
1023 
1024                         blk_queue_exit(q);
1025 
1026                         /* sort new bios into those for a lower level
1027                          * and those for the same level
1028                          */
1029                         bio_list_init(&lower);
1030                         bio_list_init(&same);
1031                         while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
1032                                 if (q == bio->bi_disk->queue)
1033                                         bio_list_add(&same, bio);
1034                                 else
1035                                         bio_list_add(&lower, bio);
1036                         /* now assemble so we handle the lowest level first */
1037                         bio_list_merge(&bio_list_on_stack[0], &lower);
1038                         bio_list_merge(&bio_list_on_stack[0], &same);
1039                         bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
1040                 } else {
1041                         if (unlikely(!blk_queue_dying(q) &&
1042                                         (bio->bi_opf & REQ_NOWAIT)))
1043                                 bio_wouldblock_error(bio);
1044                         else
1045                                 bio_io_error(bio);
1046                 }
1047                 bio = bio_list_pop(&bio_list_on_stack[0]);
1048         } while (bio);
1049         current->bio_list = NULL; /* deactivate */
1050 
1051 out:
1052         return ret;
1053 }
1054 EXPORT_SYMBOL(generic_make_request);
1055 
1056 /**
1057  * direct_make_request - hand a buffer directly to its device driver for I/O
1058  * @bio:  The bio describing the location in memory and on the device.
1059  *
1060  * This function behaves like generic_make_request(), but does not protect
1061  * against recursion.  Must only be used if the called driver is known
1062  * to not call generic_make_request (or direct_make_request) again from
1063  * its make_request function.  (Calling direct_make_request again from
1064  * a workqueue is perfectly fine as that doesn't recurse).
1065  */
1066 blk_qc_t direct_make_request(struct bio *bio)
1067 {
1068         struct request_queue *q = bio->bi_disk->queue;
1069         bool nowait = bio->bi_opf & REQ_NOWAIT;
1070         blk_qc_t ret;
1071 
1072         if (!generic_make_request_checks(bio))
1073                 return BLK_QC_T_NONE;
1074 
1075         if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
1076                 if (nowait && !blk_queue_dying(q))
1077                         bio->bi_status = BLK_STS_AGAIN;
1078                 else
1079                         bio->bi_status = BLK_STS_IOERR;
1080                 bio_endio(bio);
1081                 return BLK_QC_T_NONE;
1082         }
1083 
1084         ret = q->make_request_fn(q, bio);
1085         blk_queue_exit(q);
1086         return ret;
1087 }
1088 EXPORT_SYMBOL_GPL(direct_make_request);
1089 
1090 /**
1091  * submit_bio - submit a bio to the block device layer for I/O
1092  * @bio: The &struct bio which describes the I/O
1093  *
1094  * submit_bio() is very similar in purpose to generic_make_request(), and
1095  * uses that function to do most of the work. Both are fairly rough
1096  * interfaces; @bio must be presetup and ready for I/O.
1097  *
1098  */
1099 blk_qc_t submit_bio(struct bio *bio)
1100 {
1101         /*
1102          * If it's a regular read/write or a barrier with data attached,
1103          * go through the normal accounting stuff before submission.
1104          */
1105         if (bio_has_data(bio)) {
1106                 unsigned int count;
1107 
1108                 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
1109                         count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
1110                 else
1111                         count = bio_sectors(bio);
1112 
1113                 if (op_is_write(bio_op(bio))) {
1114                         count_vm_events(PGPGOUT, count);
1115                 } else {
1116                         task_io_account_read(bio->bi_iter.bi_size);
1117                         count_vm_events(PGPGIN, count);
1118                 }
1119 
1120                 if (unlikely(block_dump)) {
1121                         char b[BDEVNAME_SIZE];
1122                         printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1123                         current->comm, task_pid_nr(current),
1124                                 op_is_write(bio_op(bio)) ? "WRITE" : "READ",
1125                                 (unsigned long long)bio->bi_iter.bi_sector,
1126                                 bio_devname(bio, b), count);
1127                 }
1128         }
1129 
1130         return generic_make_request(bio);
1131 }
1132 EXPORT_SYMBOL(submit_bio);
1133 
1134 /**
1135  * blk_cloned_rq_check_limits - Helper function to check a cloned request
1136  *                              for new the queue limits
1137  * @q:  the queue
1138  * @rq: the request being checked
1139  *
1140  * Description:
1141  *    @rq may have been made based on weaker limitations of upper-level queues
1142  *    in request stacking drivers, and it may violate the limitation of @q.
1143  *    Since the block layer and the underlying device driver trust @rq
1144  *    after it is inserted to @q, it should be checked against @q before
1145  *    the insertion using this generic function.
1146  *
1147  *    Request stacking drivers like request-based dm may change the queue
1148  *    limits when retrying requests on other queues. Those requests need
1149  *    to be checked against the new queue limits again during dispatch.
1150  */
1151 static int blk_cloned_rq_check_limits(struct request_queue *q,
1152                                       struct request *rq)
1153 {
1154         if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
1155                 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
1156                         __func__, blk_rq_sectors(rq),
1157                         blk_queue_get_max_sectors(q, req_op(rq)));
1158                 return -EIO;
1159         }
1160 
1161         /*
1162          * queue's settings related to segment counting like q->bounce_pfn
1163          * may differ from that of other stacking queues.
1164          * Recalculate it to check the request correctly on this queue's
1165          * limitation.
1166          */
1167         blk_recalc_rq_segments(rq);
1168         if (rq->nr_phys_segments > queue_max_segments(q)) {
1169                 printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
1170                         __func__, rq->nr_phys_segments, queue_max_segments(q));
1171                 return -EIO;
1172         }
1173 
1174         return 0;
1175 }
1176 
1177 /**
1178  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1179  * @q:  the queue to submit the request
1180  * @rq: the request being queued
1181  */
1182 blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1183 {
1184         if (blk_cloned_rq_check_limits(q, rq))
1185                 return BLK_STS_IOERR;
1186 
1187         if (rq->rq_disk &&
1188             should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
1189                 return BLK_STS_IOERR;
1190 
1191         if (blk_queue_io_stat(q))
1192                 blk_account_io_start(rq, true);
1193 
1194         /*
1195          * Since we have a scheduler attached on the top device,
1196          * bypass a potential scheduler on the bottom device for
1197          * insert.
1198          */
1199         return blk_mq_request_issue_directly(rq, true);
1200 }
1201 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1202 
1203 /**
1204  * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1205  * @rq: request to examine
1206  *
1207  * Description:
1208  *     A request could be merge of IOs which require different failure
1209  *     handling.  This function determines the number of bytes which
1210  *     can be failed from the beginning of the request without
1211  *     crossing into area which need to be retried further.
1212  *
1213  * Return:
1214  *     The number of bytes to fail.
1215  */
1216 unsigned int blk_rq_err_bytes(const struct request *rq)
1217 {
1218         unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1219         unsigned int bytes = 0;
1220         struct bio *bio;
1221 
1222         if (!(rq->rq_flags & RQF_MIXED_MERGE))
1223                 return blk_rq_bytes(rq);
1224 
1225         /*
1226          * Currently the only 'mixing' which can happen is between
1227          * different fastfail types.  We can safely fail portions
1228          * which have all the failfast bits that the first one has -
1229          * the ones which are at least as eager to fail as the first
1230          * one.
1231          */
1232         for (bio = rq->bio; bio; bio = bio->bi_next) {
1233                 if ((bio->bi_opf & ff) != ff)
1234                         break;
1235                 bytes += bio->bi_iter.bi_size;
1236         }
1237 
1238         /* this could lead to infinite loop */
1239         BUG_ON(blk_rq_bytes(rq) && !bytes);
1240         return bytes;
1241 }
1242 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
1243 
1244 void blk_account_io_completion(struct request *req, unsigned int bytes)
1245 {
1246         if (blk_do_io_stat(req)) {
1247                 const int sgrp = op_stat_group(req_op(req));
1248                 struct hd_struct *part;
1249 
1250                 part_stat_lock();
1251                 part = req->part;
1252                 part_stat_add(part, sectors[sgrp], bytes >> 9);
1253                 part_stat_unlock();
1254         }
1255 }
1256 
1257 void blk_account_io_done(struct request *req, u64 now)
1258 {
1259         /*
1260          * Account IO completion.  flush_rq isn't accounted as a
1261          * normal IO on queueing nor completion.  Accounting the
1262          * containing request is enough.
1263          */
1264         if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
1265                 const int sgrp = op_stat_group(req_op(req));
1266                 struct hd_struct *part;
1267 
1268                 part_stat_lock();
1269                 part = req->part;
1270 
1271                 update_io_ticks(part, jiffies);
1272                 part_stat_inc(part, ios[sgrp]);
1273                 part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);
1274                 part_stat_add(part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns));
1275                 part_dec_in_flight(req->q, part, rq_data_dir(req));
1276 
1277                 hd_struct_put(part);
1278                 part_stat_unlock();
1279         }
1280 }
1281 
1282 void blk_account_io_start(struct request *rq, bool new_io)
1283 {
1284         struct hd_struct *part;
1285         int rw = rq_data_dir(rq);
1286 
1287         if (!blk_do_io_stat(rq))
1288                 return;
1289 
1290         part_stat_lock();
1291 
1292         if (!new_io) {
1293                 part = rq->part;
1294                 part_stat_inc(part, merges[rw]);
1295         } else {
1296                 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
1297                 if (!hd_struct_try_get(part)) {
1298                         /*
1299                          * The partition is already being removed,
1300                          * the request will be accounted on the disk only
1301                          *
1302                          * We take a reference on disk->part0 although that
1303                          * partition will never be deleted, so we can treat
1304                          * it as any other partition.
1305                          */
1306                         part = &rq->rq_disk->part0;
1307                         hd_struct_get(part);
1308                 }
1309                 part_inc_in_flight(rq->q, part, rw);
1310                 rq->part = part;
1311         }
1312 
1313         update_io_ticks(part, jiffies);
1314 
1315         part_stat_unlock();
1316 }
1317 
1318 /*
1319  * Steal bios from a request and add them to a bio list.
1320  * The request must not have been partially completed before.
1321  */
1322 void blk_steal_bios(struct bio_list *list, struct request *rq)
1323 {
1324         if (rq->bio) {
1325                 if (list->tail)
1326                         list->tail->bi_next = rq->bio;
1327                 else
1328                         list->head = rq->bio;
1329                 list->tail = rq->biotail;
1330 
1331                 rq->bio = NULL;
1332                 rq->biotail = NULL;
1333         }
1334 
1335         rq->__data_len = 0;
1336 }
1337 EXPORT_SYMBOL_GPL(blk_steal_bios);
1338 
1339 /**
1340  * blk_update_request - Special helper function for request stacking drivers
1341  * @req:      the request being processed
1342  * @error:    block status code
1343  * @nr_bytes: number of bytes to complete @req
1344  *
1345  * Description:
1346  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
1347  *     the request structure even if @req doesn't have leftover.
1348  *     If @req has leftover, sets it up for the next range of segments.
1349  *
1350  *     This special helper function is only for request stacking drivers
1351  *     (e.g. request-based dm) so that they can handle partial completion.
1352  *     Actual device drivers should use blk_end_request instead.
1353  *
1354  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
1355  *     %false return from this function.
1356  *
1357  * Note:
1358  *      The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
1359  *      blk_rq_bytes() and in blk_update_request().
1360  *
1361  * Return:
1362  *     %false - this request doesn't have any more data
1363  *     %true  - this request has more data
1364  **/
1365 bool blk_update_request(struct request *req, blk_status_t error,
1366                 unsigned int nr_bytes)
1367 {
1368         int total_bytes;
1369 
1370         trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
1371 
1372         if (!req->bio)
1373                 return false;
1374 
1375         if (unlikely(error && !blk_rq_is_passthrough(req) &&
1376                      !(req->rq_flags & RQF_QUIET)))
1377                 print_req_error(req, error);
1378 
1379         blk_account_io_completion(req, nr_bytes);
1380 
1381         total_bytes = 0;
1382         while (req->bio) {
1383                 struct bio *bio = req->bio;
1384                 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
1385 
1386                 if (bio_bytes == bio->bi_iter.bi_size)
1387                         req->bio = bio->bi_next;
1388 
1389                 /* Completion has already been traced */
1390                 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1391                 req_bio_endio(req, bio, bio_bytes, error);
1392 
1393                 total_bytes += bio_bytes;
1394                 nr_bytes -= bio_bytes;
1395 
1396                 if (!nr_bytes)
1397                         break;
1398         }
1399 
1400         /*
1401          * completely done
1402          */
1403         if (!req->bio) {
1404                 /*
1405                  * Reset counters so that the request stacking driver
1406                  * can find how many bytes remain in the request
1407                  * later.
1408                  */
1409                 req->__data_len = 0;
1410                 return false;
1411         }
1412 
1413         req->__data_len -= total_bytes;
1414 
1415         /* update sector only for requests with clear definition of sector */
1416         if (!blk_rq_is_passthrough(req))
1417                 req->__sector += total_bytes >> 9;
1418 
1419         /* mixed attributes always follow the first bio */
1420         if (req->rq_flags & RQF_MIXED_MERGE) {
1421                 req->cmd_flags &= ~REQ_FAILFAST_MASK;
1422                 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
1423         }
1424 
1425         if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
1426                 /*
1427                  * If total number of sectors is less than the first segment
1428                  * size, something has gone terribly wrong.
1429                  */
1430                 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
1431                         blk_dump_rq_flags(req, "request botched");
1432                         req->__data_len = blk_rq_cur_bytes(req);
1433                 }
1434 
1435                 /* recalculate the number of segments */
1436                 blk_recalc_rq_segments(req);
1437         }
1438 
1439         return true;
1440 }
1441 EXPORT_SYMBOL_GPL(blk_update_request);
1442 
1443 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
1444                      struct bio *bio)
1445 {
1446         if (bio_has_data(bio))
1447                 rq->nr_phys_segments = bio_phys_segments(q, bio);
1448         else if (bio_op(bio) == REQ_OP_DISCARD)
1449                 rq->nr_phys_segments = 1;
1450 
1451         rq->__data_len = bio->bi_iter.bi_size;
1452         rq->bio = rq->biotail = bio;
1453 
1454         if (bio->bi_disk)
1455                 rq->rq_disk = bio->bi_disk;
1456 }
1457 
1458 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1459 /**
1460  * rq_flush_dcache_pages - Helper function to flush all pages in a request
1461  * @rq: the request to be flushed
1462  *
1463  * Description:
1464  *     Flush all pages in @rq.
1465  */
1466 void rq_flush_dcache_pages(struct request *rq)
1467 {
1468         struct req_iterator iter;
1469         struct bio_vec bvec;
1470 
1471         rq_for_each_segment(bvec, rq, iter)
1472                 flush_dcache_page(bvec.bv_page);
1473 }
1474 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
1475 #endif
1476 
1477 /**
1478  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
1479  * @q : the queue of the device being checked
1480  *
1481  * Description:
1482  *    Check if underlying low-level drivers of a device are busy.
1483  *    If the drivers want to export their busy state, they must set own
1484  *    exporting function using blk_queue_lld_busy() first.
1485  *
1486  *    Basically, this function is used only by request stacking drivers
1487  *    to stop dispatching requests to underlying devices when underlying
1488  *    devices are busy.  This behavior helps more I/O merging on the queue
1489  *    of the request stacking driver and prevents I/O throughput regression
1490  *    on burst I/O load.
1491  *
1492  * Return:
1493  *    0 - Not busy (The request stacking driver should dispatch request)
1494  *    1 - Busy (The request stacking driver should stop dispatching request)
1495  */
1496 int blk_lld_busy(struct request_queue *q)
1497 {
1498         if (queue_is_mq(q) && q->mq_ops->busy)
1499                 return q->mq_ops->busy(q);
1500 
1501         return 0;
1502 }
1503 EXPORT_SYMBOL_GPL(blk_lld_busy);
1504 
1505 /**
1506  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
1507  * @rq: the clone request to be cleaned up
1508  *
1509  * Description:
1510  *     Free all bios in @rq for a cloned request.
1511  */
1512 void blk_rq_unprep_clone(struct request *rq)
1513 {
1514         struct bio *bio;
1515 
1516         while ((bio = rq->bio) != NULL) {
1517                 rq->bio = bio->bi_next;
1518 
1519                 bio_put(bio);
1520         }
1521 }
1522 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
1523 
1524 /*
1525  * Copy attributes of the original request to the clone request.
1526  * The actual data parts (e.g. ->cmd, ->sense) are not copied.
1527  */
1528 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
1529 {
1530         dst->__sector = blk_rq_pos(src);
1531         dst->__data_len = blk_rq_bytes(src);
1532         if (src->rq_flags & RQF_SPECIAL_PAYLOAD) {
1533                 dst->rq_flags |= RQF_SPECIAL_PAYLOAD;
1534                 dst->special_vec = src->special_vec;
1535         }
1536         dst->nr_phys_segments = src->nr_phys_segments;
1537         dst->ioprio = src->ioprio;
1538         dst->extra_len = src->extra_len;
1539 }
1540 
1541 /**
1542  * blk_rq_prep_clone - Helper function to setup clone request
1543  * @rq: the request to be setup
1544  * @rq_src: original request to be cloned
1545  * @bs: bio_set that bios for clone are allocated from
1546  * @gfp_mask: memory allocation mask for bio
1547  * @bio_ctr: setup function to be called for each clone bio.
1548  *           Returns %0 for success, non %0 for failure.
1549  * @data: private data to be passed to @bio_ctr
1550  *
1551  * Description:
1552  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
1553  *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
1554  *     are not copied, and copying such parts is the caller's responsibility.
1555  *     Also, pages which the original bios are pointing to are not copied
1556  *     and the cloned bios just point same pages.
1557  *     So cloned bios must be completed before original bios, which means
1558  *     the caller must complete @rq before @rq_src.
1559  */
1560 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
1561                       struct bio_set *bs, gfp_t gfp_mask,
1562                       int (*bio_ctr)(struct bio *, struct bio *, void *),
1563                       void *data)
1564 {
1565         struct bio *bio, *bio_src;
1566 
1567         if (!bs)
1568                 bs = &fs_bio_set;
1569 
1570         __rq_for_each_bio(bio_src, rq_src) {
1571                 bio = bio_clone_fast(bio_src, gfp_mask, bs);
1572                 if (!bio)
1573                         goto free_and_out;
1574 
1575                 if (bio_ctr && bio_ctr(bio, bio_src, data))
1576                         goto free_and_out;
1577 
1578                 if (rq->bio) {
1579                         rq->biotail->bi_next = bio;
1580                         rq->biotail = bio;
1581                 } else
1582                         rq->bio = rq->biotail = bio;
1583         }
1584 
1585         __blk_rq_prep_clone(rq, rq_src);
1586 
1587         return 0;
1588 
1589 free_and_out:
1590         if (bio)
1591                 bio_put(bio);
1592         blk_rq_unprep_clone(rq);
1593 
1594         return -ENOMEM;
1595 }
1596 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
1597 
1598 int kblockd_schedule_work(struct work_struct *work)
1599 {
1600         return queue_work(kblockd_workqueue, work);
1601 }
1602 EXPORT_SYMBOL(kblockd_schedule_work);
1603 
1604 int kblockd_schedule_work_on(int cpu, struct work_struct *work)
1605 {
1606         return queue_work_on(cpu, kblockd_workqueue, work);
1607 }
1608 EXPORT_SYMBOL(kblockd_schedule_work_on);
1609 
1610 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
1611                                 unsigned long delay)
1612 {
1613         return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
1614 }
1615 EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
1616 
1617 /**
1618  * blk_start_plug - initialize blk_plug and track it inside the task_struct
1619  * @plug:       The &struct blk_plug that needs to be initialized
1620  *
1621  * Description:
1622  *   blk_start_plug() indicates to the block layer an intent by the caller
1623  *   to submit multiple I/O requests in a batch.  The block layer may use
1624  *   this hint to defer submitting I/Os from the caller until blk_finish_plug()
1625  *   is called.  However, the block layer may choose to submit requests
1626  *   before a call to blk_finish_plug() if the number of queued I/Os
1627  *   exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than
1628  *   %BLK_PLUG_FLUSH_SIZE.  The queued I/Os may also be submitted early if
1629  *   the task schedules (see below).
1630  *
1631  *   Tracking blk_plug inside the task_struct will help with auto-flushing the
1632  *   pending I/O should the task end up blocking between blk_start_plug() and
1633  *   blk_finish_plug(). This is important from a performance perspective, but
1634  *   also ensures that we don't deadlock. For instance, if the task is blocking
1635  *   for a memory allocation, memory reclaim could end up wanting to free a
1636  *   page belonging to that request that is currently residing in our private
1637  *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
1638  *   this kind of deadlock.
1639  */
1640 void blk_start_plug(struct blk_plug *plug)
1641 {
1642         struct task_struct *tsk = current;
1643 
1644         /*
1645          * If this is a nested plug, don't actually assign it.
1646          */
1647         if (tsk->plug)
1648                 return;
1649 
1650         INIT_LIST_HEAD(&plug->mq_list);
1651         INIT_LIST_HEAD(&plug->cb_list);
1652         plug->rq_count = 0;
1653         plug->multiple_queues = false;
1654 
1655         /*
1656          * Store ordering should not be needed here, since a potential
1657          * preempt will imply a full memory barrier
1658          */
1659         tsk->plug = plug;
1660 }
1661 EXPORT_SYMBOL(blk_start_plug);
1662 
1663 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
1664 {
1665         LIST_HEAD(callbacks);
1666 
1667         while (!list_empty(&plug->cb_list)) {
1668                 list_splice_init(&plug->cb_list, &callbacks);
1669 
1670                 while (!list_empty(&callbacks)) {
1671                         struct blk_plug_cb *cb = list_first_entry(&callbacks,
1672                                                           struct blk_plug_cb,
1673                                                           list);
1674                         list_del(&cb->list);
1675                         cb->callback(cb, from_schedule);
1676                 }
1677         }
1678 }
1679 
1680 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
1681                                       int size)
1682 {
1683         struct blk_plug *plug = current->plug;
1684         struct blk_plug_cb *cb;
1685 
1686         if (!plug)
1687                 return NULL;
1688 
1689         list_for_each_entry(cb, &plug->cb_list, list)
1690                 if (cb->callback == unplug && cb->data == data)
1691                         return cb;
1692 
1693         /* Not currently on the callback list */
1694         BUG_ON(size < sizeof(*cb));
1695         cb = kzalloc(size, GFP_ATOMIC);
1696         if (cb) {
1697                 cb->data = data;
1698                 cb->callback = unplug;
1699                 list_add(&cb->list, &plug->cb_list);
1700         }
1701         return cb;
1702 }
1703 EXPORT_SYMBOL(blk_check_plugged);
1704 
1705 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1706 {
1707         flush_plug_callbacks(plug, from_schedule);
1708 
1709         if (!list_empty(&plug->mq_list))
1710                 blk_mq_flush_plug_list(plug, from_schedule);
1711 }
1712 
1713 /**
1714  * blk_finish_plug - mark the end of a batch of submitted I/O
1715  * @plug:       The &struct blk_plug passed to blk_start_plug()
1716  *
1717  * Description:
1718  * Indicate that a batch of I/O submissions is complete.  This function
1719  * must be paired with an initial call to blk_start_plug().  The intent
1720  * is to allow the block layer to optimize I/O submission.  See the
1721  * documentation for blk_start_plug() for more information.
1722  */
1723 void blk_finish_plug(struct blk_plug *plug)
1724 {
1725         if (plug != current->plug)
1726                 return;
1727         blk_flush_plug_list(plug, false);
1728 
1729         current->plug = NULL;
1730 }
1731 EXPORT_SYMBOL(blk_finish_plug);
1732 
1733 int __init blk_dev_init(void)
1734 {
1735         BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
1736         BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1737                         FIELD_SIZEOF(struct request, cmd_flags));
1738         BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1739                         FIELD_SIZEOF(struct bio, bi_opf));
1740 
1741         /* used for unplugging and affects IO latency/throughput - HIGHPRI */
1742         kblockd_workqueue = alloc_workqueue("kblockd",
1743                                             WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1744         if (!kblockd_workqueue)
1745                 panic("Failed to create kblockd\n");
1746 
1747         blk_requestq_cachep = kmem_cache_create("request_queue",
1748                         sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
1749 
1750 #ifdef CONFIG_DEBUG_FS
1751         blk_debugfs_root = debugfs_create_dir("block", NULL);
1752 #endif
1753 
1754         return 0;
1755 }
1756 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp