~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/block/blk-core.c

Version: ~ [ linux-5.16-rc1 ] ~ [ linux-5.15.2 ] ~ [ linux-5.14.18 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.79 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.159 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.217 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.255 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.290 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.292 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (C) 1991, 1992 Linus Torvalds
  3  * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
  4  * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
  5  * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
  6  * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
  7  *      -  July2000
  8  * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
  9  */
 10 
 11 /*
 12  * This handles all read/write requests to block devices
 13  */
 14 #include <linux/kernel.h>
 15 #include <linux/module.h>
 16 #include <linux/backing-dev.h>
 17 #include <linux/bio.h>
 18 #include <linux/blkdev.h>
 19 #include <linux/blk-mq.h>
 20 #include <linux/highmem.h>
 21 #include <linux/mm.h>
 22 #include <linux/kernel_stat.h>
 23 #include <linux/string.h>
 24 #include <linux/init.h>
 25 #include <linux/completion.h>
 26 #include <linux/slab.h>
 27 #include <linux/swap.h>
 28 #include <linux/writeback.h>
 29 #include <linux/task_io_accounting_ops.h>
 30 #include <linux/fault-inject.h>
 31 #include <linux/list_sort.h>
 32 #include <linux/delay.h>
 33 #include <linux/ratelimit.h>
 34 #include <linux/pm_runtime.h>
 35 #include <linux/blk-cgroup.h>
 36 #include <linux/debugfs.h>
 37 
 38 #define CREATE_TRACE_POINTS
 39 #include <trace/events/block.h>
 40 
 41 #include "blk.h"
 42 #include "blk-mq.h"
 43 #include "blk-mq-sched.h"
 44 #include "blk-wbt.h"
 45 
 46 #ifdef CONFIG_DEBUG_FS
 47 struct dentry *blk_debugfs_root;
 48 #endif
 49 
 50 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
 51 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
 52 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
 53 EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
 54 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
 55 
 56 DEFINE_IDA(blk_queue_ida);
 57 
 58 /*
 59  * For the allocated request tables
 60  */
 61 struct kmem_cache *request_cachep;
 62 
 63 /*
 64  * For queue allocation
 65  */
 66 struct kmem_cache *blk_requestq_cachep;
 67 
 68 /*
 69  * Controlling structure to kblockd
 70  */
 71 static struct workqueue_struct *kblockd_workqueue;
 72 
 73 static void blk_clear_congested(struct request_list *rl, int sync)
 74 {
 75 #ifdef CONFIG_CGROUP_WRITEBACK
 76         clear_wb_congested(rl->blkg->wb_congested, sync);
 77 #else
 78         /*
 79          * If !CGROUP_WRITEBACK, all blkg's map to bdi->wb and we shouldn't
 80          * flip its congestion state for events on other blkcgs.
 81          */
 82         if (rl == &rl->q->root_rl)
 83                 clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
 84 #endif
 85 }
 86 
 87 static void blk_set_congested(struct request_list *rl, int sync)
 88 {
 89 #ifdef CONFIG_CGROUP_WRITEBACK
 90         set_wb_congested(rl->blkg->wb_congested, sync);
 91 #else
 92         /* see blk_clear_congested() */
 93         if (rl == &rl->q->root_rl)
 94                 set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
 95 #endif
 96 }
 97 
 98 void blk_queue_congestion_threshold(struct request_queue *q)
 99 {
100         int nr;
101 
102         nr = q->nr_requests - (q->nr_requests / 8) + 1;
103         if (nr > q->nr_requests)
104                 nr = q->nr_requests;
105         q->nr_congestion_on = nr;
106 
107         nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
108         if (nr < 1)
109                 nr = 1;
110         q->nr_congestion_off = nr;
111 }
112 
113 void blk_rq_init(struct request_queue *q, struct request *rq)
114 {
115         memset(rq, 0, sizeof(*rq));
116 
117         INIT_LIST_HEAD(&rq->queuelist);
118         INIT_LIST_HEAD(&rq->timeout_list);
119         rq->cpu = -1;
120         rq->q = q;
121         rq->__sector = (sector_t) -1;
122         INIT_HLIST_NODE(&rq->hash);
123         RB_CLEAR_NODE(&rq->rb_node);
124         rq->tag = -1;
125         rq->internal_tag = -1;
126         rq->start_time = jiffies;
127         set_start_time_ns(rq);
128         rq->part = NULL;
129 }
130 EXPORT_SYMBOL(blk_rq_init);
131 
132 static const struct {
133         int             errno;
134         const char      *name;
135 } blk_errors[] = {
136         [BLK_STS_OK]            = { 0,          "" },
137         [BLK_STS_NOTSUPP]       = { -EOPNOTSUPP, "operation not supported" },
138         [BLK_STS_TIMEOUT]       = { -ETIMEDOUT, "timeout" },
139         [BLK_STS_NOSPC]         = { -ENOSPC,    "critical space allocation" },
140         [BLK_STS_TRANSPORT]     = { -ENOLINK,   "recoverable transport" },
141         [BLK_STS_TARGET]        = { -EREMOTEIO, "critical target" },
142         [BLK_STS_NEXUS]         = { -EBADE,     "critical nexus" },
143         [BLK_STS_MEDIUM]        = { -ENODATA,   "critical medium" },
144         [BLK_STS_PROTECTION]    = { -EILSEQ,    "protection" },
145         [BLK_STS_RESOURCE]      = { -ENOMEM,    "kernel resource" },
146         [BLK_STS_AGAIN]         = { -EAGAIN,    "nonblocking retry" },
147 
148         /* device mapper special case, should not leak out: */
149         [BLK_STS_DM_REQUEUE]    = { -EREMCHG, "dm internal retry" },
150 
151         /* everything else not covered above: */
152         [BLK_STS_IOERR]         = { -EIO,       "I/O" },
153 };
154 
155 blk_status_t errno_to_blk_status(int errno)
156 {
157         int i;
158 
159         for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
160                 if (blk_errors[i].errno == errno)
161                         return (__force blk_status_t)i;
162         }
163 
164         return BLK_STS_IOERR;
165 }
166 EXPORT_SYMBOL_GPL(errno_to_blk_status);
167 
168 int blk_status_to_errno(blk_status_t status)
169 {
170         int idx = (__force int)status;
171 
172         if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
173                 return -EIO;
174         return blk_errors[idx].errno;
175 }
176 EXPORT_SYMBOL_GPL(blk_status_to_errno);
177 
178 static void print_req_error(struct request *req, blk_status_t status)
179 {
180         int idx = (__force int)status;
181 
182         if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
183                 return;
184 
185         printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n",
186                            __func__, blk_errors[idx].name, req->rq_disk ?
187                            req->rq_disk->disk_name : "?",
188                            (unsigned long long)blk_rq_pos(req));
189 }
190 
191 static void req_bio_endio(struct request *rq, struct bio *bio,
192                           unsigned int nbytes, blk_status_t error)
193 {
194         if (error)
195                 bio->bi_status = error;
196 
197         if (unlikely(rq->rq_flags & RQF_QUIET))
198                 bio_set_flag(bio, BIO_QUIET);
199 
200         bio_advance(bio, nbytes);
201 
202         /* don't actually finish bio if it's part of flush sequence */
203         if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
204                 bio_endio(bio);
205 }
206 
207 void blk_dump_rq_flags(struct request *rq, char *msg)
208 {
209         printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
210                 rq->rq_disk ? rq->rq_disk->disk_name : "?",
211                 (unsigned long long) rq->cmd_flags);
212 
213         printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
214                (unsigned long long)blk_rq_pos(rq),
215                blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
216         printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
217                rq->bio, rq->biotail, blk_rq_bytes(rq));
218 }
219 EXPORT_SYMBOL(blk_dump_rq_flags);
220 
221 static void blk_delay_work(struct work_struct *work)
222 {
223         struct request_queue *q;
224 
225         q = container_of(work, struct request_queue, delay_work.work);
226         spin_lock_irq(q->queue_lock);
227         __blk_run_queue(q);
228         spin_unlock_irq(q->queue_lock);
229 }
230 
231 /**
232  * blk_delay_queue - restart queueing after defined interval
233  * @q:          The &struct request_queue in question
234  * @msecs:      Delay in msecs
235  *
236  * Description:
237  *   Sometimes queueing needs to be postponed for a little while, to allow
238  *   resources to come back. This function will make sure that queueing is
239  *   restarted around the specified time.
240  */
241 void blk_delay_queue(struct request_queue *q, unsigned long msecs)
242 {
243         lockdep_assert_held(q->queue_lock);
244         WARN_ON_ONCE(q->mq_ops);
245 
246         if (likely(!blk_queue_dead(q)))
247                 queue_delayed_work(kblockd_workqueue, &q->delay_work,
248                                    msecs_to_jiffies(msecs));
249 }
250 EXPORT_SYMBOL(blk_delay_queue);
251 
252 /**
253  * blk_start_queue_async - asynchronously restart a previously stopped queue
254  * @q:    The &struct request_queue in question
255  *
256  * Description:
257  *   blk_start_queue_async() will clear the stop flag on the queue, and
258  *   ensure that the request_fn for the queue is run from an async
259  *   context.
260  **/
261 void blk_start_queue_async(struct request_queue *q)
262 {
263         lockdep_assert_held(q->queue_lock);
264         WARN_ON_ONCE(q->mq_ops);
265 
266         queue_flag_clear(QUEUE_FLAG_STOPPED, q);
267         blk_run_queue_async(q);
268 }
269 EXPORT_SYMBOL(blk_start_queue_async);
270 
271 /**
272  * blk_start_queue - restart a previously stopped queue
273  * @q:    The &struct request_queue in question
274  *
275  * Description:
276  *   blk_start_queue() will clear the stop flag on the queue, and call
277  *   the request_fn for the queue if it was in a stopped state when
278  *   entered. Also see blk_stop_queue().
279  **/
280 void blk_start_queue(struct request_queue *q)
281 {
282         lockdep_assert_held(q->queue_lock);
283         WARN_ON(!in_interrupt() && !irqs_disabled());
284         WARN_ON_ONCE(q->mq_ops);
285 
286         queue_flag_clear(QUEUE_FLAG_STOPPED, q);
287         __blk_run_queue(q);
288 }
289 EXPORT_SYMBOL(blk_start_queue);
290 
291 /**
292  * blk_stop_queue - stop a queue
293  * @q:    The &struct request_queue in question
294  *
295  * Description:
296  *   The Linux block layer assumes that a block driver will consume all
297  *   entries on the request queue when the request_fn strategy is called.
298  *   Often this will not happen, because of hardware limitations (queue
299  *   depth settings). If a device driver gets a 'queue full' response,
300  *   or if it simply chooses not to queue more I/O at one point, it can
301  *   call this function to prevent the request_fn from being called until
302  *   the driver has signalled it's ready to go again. This happens by calling
303  *   blk_start_queue() to restart queue operations.
304  **/
305 void blk_stop_queue(struct request_queue *q)
306 {
307         lockdep_assert_held(q->queue_lock);
308         WARN_ON_ONCE(q->mq_ops);
309 
310         cancel_delayed_work(&q->delay_work);
311         queue_flag_set(QUEUE_FLAG_STOPPED, q);
312 }
313 EXPORT_SYMBOL(blk_stop_queue);
314 
315 /**
316  * blk_sync_queue - cancel any pending callbacks on a queue
317  * @q: the queue
318  *
319  * Description:
320  *     The block layer may perform asynchronous callback activity
321  *     on a queue, such as calling the unplug function after a timeout.
322  *     A block device may call blk_sync_queue to ensure that any
323  *     such activity is cancelled, thus allowing it to release resources
324  *     that the callbacks might use. The caller must already have made sure
325  *     that its ->make_request_fn will not re-add plugging prior to calling
326  *     this function.
327  *
328  *     This function does not cancel any asynchronous activity arising
329  *     out of elevator or throttling code. That would require elevator_exit()
330  *     and blkcg_exit_queue() to be called with queue lock initialized.
331  *
332  */
333 void blk_sync_queue(struct request_queue *q)
334 {
335         del_timer_sync(&q->timeout);
336 
337         if (q->mq_ops) {
338                 struct blk_mq_hw_ctx *hctx;
339                 int i;
340 
341                 queue_for_each_hw_ctx(q, hctx, i)
342                         cancel_delayed_work_sync(&hctx->run_work);
343         } else {
344                 cancel_delayed_work_sync(&q->delay_work);
345         }
346 }
347 EXPORT_SYMBOL(blk_sync_queue);
348 
349 /**
350  * __blk_run_queue_uncond - run a queue whether or not it has been stopped
351  * @q:  The queue to run
352  *
353  * Description:
354  *    Invoke request handling on a queue if there are any pending requests.
355  *    May be used to restart request handling after a request has completed.
356  *    This variant runs the queue whether or not the queue has been
357  *    stopped. Must be called with the queue lock held and interrupts
358  *    disabled. See also @blk_run_queue.
359  */
360 inline void __blk_run_queue_uncond(struct request_queue *q)
361 {
362         lockdep_assert_held(q->queue_lock);
363         WARN_ON_ONCE(q->mq_ops);
364 
365         if (unlikely(blk_queue_dead(q)))
366                 return;
367 
368         /*
369          * Some request_fn implementations, e.g. scsi_request_fn(), unlock
370          * the queue lock internally. As a result multiple threads may be
371          * running such a request function concurrently. Keep track of the
372          * number of active request_fn invocations such that blk_drain_queue()
373          * can wait until all these request_fn calls have finished.
374          */
375         q->request_fn_active++;
376         q->request_fn(q);
377         q->request_fn_active--;
378 }
379 EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
380 
381 /**
382  * __blk_run_queue - run a single device queue
383  * @q:  The queue to run
384  *
385  * Description:
386  *    See @blk_run_queue.
387  */
388 void __blk_run_queue(struct request_queue *q)
389 {
390         lockdep_assert_held(q->queue_lock);
391         WARN_ON_ONCE(q->mq_ops);
392 
393         if (unlikely(blk_queue_stopped(q)))
394                 return;
395 
396         __blk_run_queue_uncond(q);
397 }
398 EXPORT_SYMBOL(__blk_run_queue);
399 
400 /**
401  * blk_run_queue_async - run a single device queue in workqueue context
402  * @q:  The queue to run
403  *
404  * Description:
405  *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
406  *    of us.
407  *
408  * Note:
409  *    Since it is not allowed to run q->delay_work after blk_cleanup_queue()
410  *    has canceled q->delay_work, callers must hold the queue lock to avoid
411  *    race conditions between blk_cleanup_queue() and blk_run_queue_async().
412  */
413 void blk_run_queue_async(struct request_queue *q)
414 {
415         lockdep_assert_held(q->queue_lock);
416         WARN_ON_ONCE(q->mq_ops);
417 
418         if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
419                 mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
420 }
421 EXPORT_SYMBOL(blk_run_queue_async);
422 
423 /**
424  * blk_run_queue - run a single device queue
425  * @q: The queue to run
426  *
427  * Description:
428  *    Invoke request handling on this queue, if it has pending work to do.
429  *    May be used to restart queueing when a request has completed.
430  */
431 void blk_run_queue(struct request_queue *q)
432 {
433         unsigned long flags;
434 
435         WARN_ON_ONCE(q->mq_ops);
436 
437         spin_lock_irqsave(q->queue_lock, flags);
438         __blk_run_queue(q);
439         spin_unlock_irqrestore(q->queue_lock, flags);
440 }
441 EXPORT_SYMBOL(blk_run_queue);
442 
443 void blk_put_queue(struct request_queue *q)
444 {
445         kobject_put(&q->kobj);
446 }
447 EXPORT_SYMBOL(blk_put_queue);
448 
449 /**
450  * __blk_drain_queue - drain requests from request_queue
451  * @q: queue to drain
452  * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
453  *
454  * Drain requests from @q.  If @drain_all is set, all requests are drained.
455  * If not, only ELVPRIV requests are drained.  The caller is responsible
456  * for ensuring that no new requests which need to be drained are queued.
457  */
458 static void __blk_drain_queue(struct request_queue *q, bool drain_all)
459         __releases(q->queue_lock)
460         __acquires(q->queue_lock)
461 {
462         int i;
463 
464         lockdep_assert_held(q->queue_lock);
465         WARN_ON_ONCE(q->mq_ops);
466 
467         while (true) {
468                 bool drain = false;
469 
470                 /*
471                  * The caller might be trying to drain @q before its
472                  * elevator is initialized.
473                  */
474                 if (q->elevator)
475                         elv_drain_elevator(q);
476 
477                 blkcg_drain_queue(q);
478 
479                 /*
480                  * This function might be called on a queue which failed
481                  * driver init after queue creation or is not yet fully
482                  * active yet.  Some drivers (e.g. fd and loop) get unhappy
483                  * in such cases.  Kick queue iff dispatch queue has
484                  * something on it and @q has request_fn set.
485                  */
486                 if (!list_empty(&q->queue_head) && q->request_fn)
487                         __blk_run_queue(q);
488 
489                 drain |= q->nr_rqs_elvpriv;
490                 drain |= q->request_fn_active;
491 
492                 /*
493                  * Unfortunately, requests are queued at and tracked from
494                  * multiple places and there's no single counter which can
495                  * be drained.  Check all the queues and counters.
496                  */
497                 if (drain_all) {
498                         struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
499                         drain |= !list_empty(&q->queue_head);
500                         for (i = 0; i < 2; i++) {
501                                 drain |= q->nr_rqs[i];
502                                 drain |= q->in_flight[i];
503                                 if (fq)
504                                     drain |= !list_empty(&fq->flush_queue[i]);
505                         }
506                 }
507 
508                 if (!drain)
509                         break;
510 
511                 spin_unlock_irq(q->queue_lock);
512 
513                 msleep(10);
514 
515                 spin_lock_irq(q->queue_lock);
516         }
517 
518         /*
519          * With queue marked dead, any woken up waiter will fail the
520          * allocation path, so the wakeup chaining is lost and we're
521          * left with hung waiters. We need to wake up those waiters.
522          */
523         if (q->request_fn) {
524                 struct request_list *rl;
525 
526                 blk_queue_for_each_rl(rl, q)
527                         for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
528                                 wake_up_all(&rl->wait[i]);
529         }
530 }
531 
532 /**
533  * blk_queue_bypass_start - enter queue bypass mode
534  * @q: queue of interest
535  *
536  * In bypass mode, only the dispatch FIFO queue of @q is used.  This
537  * function makes @q enter bypass mode and drains all requests which were
538  * throttled or issued before.  On return, it's guaranteed that no request
539  * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
540  * inside queue or RCU read lock.
541  */
542 void blk_queue_bypass_start(struct request_queue *q)
543 {
544         WARN_ON_ONCE(q->mq_ops);
545 
546         spin_lock_irq(q->queue_lock);
547         q->bypass_depth++;
548         queue_flag_set(QUEUE_FLAG_BYPASS, q);
549         spin_unlock_irq(q->queue_lock);
550 
551         /*
552          * Queues start drained.  Skip actual draining till init is
553          * complete.  This avoids lenghty delays during queue init which
554          * can happen many times during boot.
555          */
556         if (blk_queue_init_done(q)) {
557                 spin_lock_irq(q->queue_lock);
558                 __blk_drain_queue(q, false);
559                 spin_unlock_irq(q->queue_lock);
560 
561                 /* ensure blk_queue_bypass() is %true inside RCU read lock */
562                 synchronize_rcu();
563         }
564 }
565 EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
566 
567 /**
568  * blk_queue_bypass_end - leave queue bypass mode
569  * @q: queue of interest
570  *
571  * Leave bypass mode and restore the normal queueing behavior.
572  *
573  * Note: although blk_queue_bypass_start() is only called for blk-sq queues,
574  * this function is called for both blk-sq and blk-mq queues.
575  */
576 void blk_queue_bypass_end(struct request_queue *q)
577 {
578         spin_lock_irq(q->queue_lock);
579         if (!--q->bypass_depth)
580                 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
581         WARN_ON_ONCE(q->bypass_depth < 0);
582         spin_unlock_irq(q->queue_lock);
583 }
584 EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
585 
586 void blk_set_queue_dying(struct request_queue *q)
587 {
588         spin_lock_irq(q->queue_lock);
589         queue_flag_set(QUEUE_FLAG_DYING, q);
590         spin_unlock_irq(q->queue_lock);
591 
592         /*
593          * When queue DYING flag is set, we need to block new req
594          * entering queue, so we call blk_freeze_queue_start() to
595          * prevent I/O from crossing blk_queue_enter().
596          */
597         blk_freeze_queue_start(q);
598 
599         if (q->mq_ops)
600                 blk_mq_wake_waiters(q);
601         else {
602                 struct request_list *rl;
603 
604                 spin_lock_irq(q->queue_lock);
605                 blk_queue_for_each_rl(rl, q) {
606                         if (rl->rq_pool) {
607                                 wake_up(&rl->wait[BLK_RW_SYNC]);
608                                 wake_up(&rl->wait[BLK_RW_ASYNC]);
609                         }
610                 }
611                 spin_unlock_irq(q->queue_lock);
612         }
613 }
614 EXPORT_SYMBOL_GPL(blk_set_queue_dying);
615 
616 /**
617  * blk_cleanup_queue - shutdown a request queue
618  * @q: request queue to shutdown
619  *
620  * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
621  * put it.  All future requests will be failed immediately with -ENODEV.
622  */
623 void blk_cleanup_queue(struct request_queue *q)
624 {
625         spinlock_t *lock = q->queue_lock;
626 
627         /* mark @q DYING, no new request or merges will be allowed afterwards */
628         mutex_lock(&q->sysfs_lock);
629         blk_set_queue_dying(q);
630         spin_lock_irq(lock);
631 
632         /*
633          * A dying queue is permanently in bypass mode till released.  Note
634          * that, unlike blk_queue_bypass_start(), we aren't performing
635          * synchronize_rcu() after entering bypass mode to avoid the delay
636          * as some drivers create and destroy a lot of queues while
637          * probing.  This is still safe because blk_release_queue() will be
638          * called only after the queue refcnt drops to zero and nothing,
639          * RCU or not, would be traversing the queue by then.
640          */
641         q->bypass_depth++;
642         queue_flag_set(QUEUE_FLAG_BYPASS, q);
643 
644         queue_flag_set(QUEUE_FLAG_NOMERGES, q);
645         queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
646         queue_flag_set(QUEUE_FLAG_DYING, q);
647         spin_unlock_irq(lock);
648         mutex_unlock(&q->sysfs_lock);
649 
650         /*
651          * Drain all requests queued before DYING marking. Set DEAD flag to
652          * prevent that q->request_fn() gets invoked after draining finished.
653          */
654         blk_freeze_queue(q);
655         spin_lock_irq(lock);
656         if (!q->mq_ops)
657                 __blk_drain_queue(q, true);
658         queue_flag_set(QUEUE_FLAG_DEAD, q);
659         spin_unlock_irq(lock);
660 
661         /* for synchronous bio-based driver finish in-flight integrity i/o */
662         blk_flush_integrity();
663 
664         /* @q won't process any more request, flush async actions */
665         del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
666         blk_sync_queue(q);
667 
668         if (q->mq_ops)
669                 blk_mq_free_queue(q);
670         percpu_ref_exit(&q->q_usage_counter);
671 
672         spin_lock_irq(lock);
673         if (q->queue_lock != &q->__queue_lock)
674                 q->queue_lock = &q->__queue_lock;
675         spin_unlock_irq(lock);
676 
677         /* @q is and will stay empty, shutdown and put */
678         blk_put_queue(q);
679 }
680 EXPORT_SYMBOL(blk_cleanup_queue);
681 
682 /* Allocate memory local to the request queue */
683 static void *alloc_request_simple(gfp_t gfp_mask, void *data)
684 {
685         struct request_queue *q = data;
686 
687         return kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
688 }
689 
690 static void free_request_simple(void *element, void *data)
691 {
692         kmem_cache_free(request_cachep, element);
693 }
694 
695 static void *alloc_request_size(gfp_t gfp_mask, void *data)
696 {
697         struct request_queue *q = data;
698         struct request *rq;
699 
700         rq = kmalloc_node(sizeof(struct request) + q->cmd_size, gfp_mask,
701                         q->node);
702         if (rq && q->init_rq_fn && q->init_rq_fn(q, rq, gfp_mask) < 0) {
703                 kfree(rq);
704                 rq = NULL;
705         }
706         return rq;
707 }
708 
709 static void free_request_size(void *element, void *data)
710 {
711         struct request_queue *q = data;
712 
713         if (q->exit_rq_fn)
714                 q->exit_rq_fn(q, element);
715         kfree(element);
716 }
717 
718 int blk_init_rl(struct request_list *rl, struct request_queue *q,
719                 gfp_t gfp_mask)
720 {
721         if (unlikely(rl->rq_pool))
722                 return 0;
723 
724         rl->q = q;
725         rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
726         rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
727         init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
728         init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
729 
730         if (q->cmd_size) {
731                 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
732                                 alloc_request_size, free_request_size,
733                                 q, gfp_mask, q->node);
734         } else {
735                 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
736                                 alloc_request_simple, free_request_simple,
737                                 q, gfp_mask, q->node);
738         }
739         if (!rl->rq_pool)
740                 return -ENOMEM;
741 
742         if (rl != &q->root_rl)
743                 WARN_ON_ONCE(!blk_get_queue(q));
744 
745         return 0;
746 }
747 
748 void blk_exit_rl(struct request_queue *q, struct request_list *rl)
749 {
750         if (rl->rq_pool) {
751                 mempool_destroy(rl->rq_pool);
752                 if (rl != &q->root_rl)
753                         blk_put_queue(q);
754         }
755 }
756 
757 struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
758 {
759         return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
760 }
761 EXPORT_SYMBOL(blk_alloc_queue);
762 
763 int blk_queue_enter(struct request_queue *q, bool nowait)
764 {
765         while (true) {
766                 int ret;
767 
768                 if (percpu_ref_tryget_live(&q->q_usage_counter))
769                         return 0;
770 
771                 if (nowait)
772                         return -EBUSY;
773 
774                 /*
775                  * read pair of barrier in blk_freeze_queue_start(),
776                  * we need to order reading __PERCPU_REF_DEAD flag of
777                  * .q_usage_counter and reading .mq_freeze_depth or
778                  * queue dying flag, otherwise the following wait may
779                  * never return if the two reads are reordered.
780                  */
781                 smp_rmb();
782 
783                 ret = wait_event_interruptible(q->mq_freeze_wq,
784                                 !atomic_read(&q->mq_freeze_depth) ||
785                                 blk_queue_dying(q));
786                 if (blk_queue_dying(q))
787                         return -ENODEV;
788                 if (ret)
789                         return ret;
790         }
791 }
792 
793 void blk_queue_exit(struct request_queue *q)
794 {
795         percpu_ref_put(&q->q_usage_counter);
796 }
797 
798 static void blk_queue_usage_counter_release(struct percpu_ref *ref)
799 {
800         struct request_queue *q =
801                 container_of(ref, struct request_queue, q_usage_counter);
802 
803         wake_up_all(&q->mq_freeze_wq);
804 }
805 
806 static void blk_rq_timed_out_timer(unsigned long data)
807 {
808         struct request_queue *q = (struct request_queue *)data;
809 
810         kblockd_schedule_work(&q->timeout_work);
811 }
812 
813 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
814 {
815         struct request_queue *q;
816 
817         q = kmem_cache_alloc_node(blk_requestq_cachep,
818                                 gfp_mask | __GFP_ZERO, node_id);
819         if (!q)
820                 return NULL;
821 
822         q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
823         if (q->id < 0)
824                 goto fail_q;
825 
826         q->bio_split = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
827         if (!q->bio_split)
828                 goto fail_id;
829 
830         q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
831         if (!q->backing_dev_info)
832                 goto fail_split;
833 
834         q->stats = blk_alloc_queue_stats();
835         if (!q->stats)
836                 goto fail_stats;
837 
838         q->backing_dev_info->ra_pages =
839                         (VM_MAX_READAHEAD * 1024) / PAGE_SIZE;
840         q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
841         q->backing_dev_info->name = "block";
842         q->node = node_id;
843 
844         setup_timer(&q->backing_dev_info->laptop_mode_wb_timer,
845                     laptop_mode_timer_fn, (unsigned long) q);
846         setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
847         INIT_LIST_HEAD(&q->queue_head);
848         INIT_LIST_HEAD(&q->timeout_list);
849         INIT_LIST_HEAD(&q->icq_list);
850 #ifdef CONFIG_BLK_CGROUP
851         INIT_LIST_HEAD(&q->blkg_list);
852 #endif
853         INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
854 
855         kobject_init(&q->kobj, &blk_queue_ktype);
856 
857         mutex_init(&q->sysfs_lock);
858         spin_lock_init(&q->__queue_lock);
859 
860         /*
861          * By default initialize queue_lock to internal lock and driver can
862          * override it later if need be.
863          */
864         q->queue_lock = &q->__queue_lock;
865 
866         /*
867          * A queue starts its life with bypass turned on to avoid
868          * unnecessary bypass on/off overhead and nasty surprises during
869          * init.  The initial bypass will be finished when the queue is
870          * registered by blk_register_queue().
871          */
872         q->bypass_depth = 1;
873         __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
874 
875         init_waitqueue_head(&q->mq_freeze_wq);
876 
877         /*
878          * Init percpu_ref in atomic mode so that it's faster to shutdown.
879          * See blk_register_queue() for details.
880          */
881         if (percpu_ref_init(&q->q_usage_counter,
882                                 blk_queue_usage_counter_release,
883                                 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
884                 goto fail_bdi;
885 
886         if (blkcg_init_queue(q))
887                 goto fail_ref;
888 
889         return q;
890 
891 fail_ref:
892         percpu_ref_exit(&q->q_usage_counter);
893 fail_bdi:
894         blk_free_queue_stats(q->stats);
895 fail_stats:
896         bdi_put(q->backing_dev_info);
897 fail_split:
898         bioset_free(q->bio_split);
899 fail_id:
900         ida_simple_remove(&blk_queue_ida, q->id);
901 fail_q:
902         kmem_cache_free(blk_requestq_cachep, q);
903         return NULL;
904 }
905 EXPORT_SYMBOL(blk_alloc_queue_node);
906 
907 /**
908  * blk_init_queue  - prepare a request queue for use with a block device
909  * @rfn:  The function to be called to process requests that have been
910  *        placed on the queue.
911  * @lock: Request queue spin lock
912  *
913  * Description:
914  *    If a block device wishes to use the standard request handling procedures,
915  *    which sorts requests and coalesces adjacent requests, then it must
916  *    call blk_init_queue().  The function @rfn will be called when there
917  *    are requests on the queue that need to be processed.  If the device
918  *    supports plugging, then @rfn may not be called immediately when requests
919  *    are available on the queue, but may be called at some time later instead.
920  *    Plugged queues are generally unplugged when a buffer belonging to one
921  *    of the requests on the queue is needed, or due to memory pressure.
922  *
923  *    @rfn is not required, or even expected, to remove all requests off the
924  *    queue, but only as many as it can handle at a time.  If it does leave
925  *    requests on the queue, it is responsible for arranging that the requests
926  *    get dealt with eventually.
927  *
928  *    The queue spin lock must be held while manipulating the requests on the
929  *    request queue; this lock will be taken also from interrupt context, so irq
930  *    disabling is needed for it.
931  *
932  *    Function returns a pointer to the initialized request queue, or %NULL if
933  *    it didn't succeed.
934  *
935  * Note:
936  *    blk_init_queue() must be paired with a blk_cleanup_queue() call
937  *    when the block device is deactivated (such as at module unload).
938  **/
939 
940 struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
941 {
942         return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
943 }
944 EXPORT_SYMBOL(blk_init_queue);
945 
946 struct request_queue *
947 blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
948 {
949         struct request_queue *q;
950 
951         q = blk_alloc_queue_node(GFP_KERNEL, node_id);
952         if (!q)
953                 return NULL;
954 
955         q->request_fn = rfn;
956         if (lock)
957                 q->queue_lock = lock;
958         if (blk_init_allocated_queue(q) < 0) {
959                 blk_cleanup_queue(q);
960                 return NULL;
961         }
962 
963         return q;
964 }
965 EXPORT_SYMBOL(blk_init_queue_node);
966 
967 static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
968 
969 
970 int blk_init_allocated_queue(struct request_queue *q)
971 {
972         WARN_ON_ONCE(q->mq_ops);
973 
974         q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size);
975         if (!q->fq)
976                 return -ENOMEM;
977 
978         if (q->init_rq_fn && q->init_rq_fn(q, q->fq->flush_rq, GFP_KERNEL))
979                 goto out_free_flush_queue;
980 
981         if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
982                 goto out_exit_flush_rq;
983 
984         INIT_WORK(&q->timeout_work, blk_timeout_work);
985         q->queue_flags          |= QUEUE_FLAG_DEFAULT;
986 
987         /*
988          * This also sets hw/phys segments, boundary and size
989          */
990         blk_queue_make_request(q, blk_queue_bio);
991 
992         q->sg_reserved_size = INT_MAX;
993 
994         /* Protect q->elevator from elevator_change */
995         mutex_lock(&q->sysfs_lock);
996 
997         /* init elevator */
998         if (elevator_init(q, NULL)) {
999                 mutex_unlock(&q->sysfs_lock);
1000                 goto out_exit_flush_rq;
1001         }
1002 
1003         mutex_unlock(&q->sysfs_lock);
1004         return 0;
1005 
1006 out_exit_flush_rq:
1007         if (q->exit_rq_fn)
1008                 q->exit_rq_fn(q, q->fq->flush_rq);
1009 out_free_flush_queue:
1010         blk_free_flush_queue(q->fq);
1011         return -ENOMEM;
1012 }
1013 EXPORT_SYMBOL(blk_init_allocated_queue);
1014 
1015 bool blk_get_queue(struct request_queue *q)
1016 {
1017         if (likely(!blk_queue_dying(q))) {
1018                 __blk_get_queue(q);
1019                 return true;
1020         }
1021 
1022         return false;
1023 }
1024 EXPORT_SYMBOL(blk_get_queue);
1025 
1026 static inline void blk_free_request(struct request_list *rl, struct request *rq)
1027 {
1028         if (rq->rq_flags & RQF_ELVPRIV) {
1029                 elv_put_request(rl->q, rq);
1030                 if (rq->elv.icq)
1031                         put_io_context(rq->elv.icq->ioc);
1032         }
1033 
1034         mempool_free(rq, rl->rq_pool);
1035 }
1036 
1037 /*
1038  * ioc_batching returns true if the ioc is a valid batching request and
1039  * should be given priority access to a request.
1040  */
1041 static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
1042 {
1043         if (!ioc)
1044                 return 0;
1045 
1046         /*
1047          * Make sure the process is able to allocate at least 1 request
1048          * even if the batch times out, otherwise we could theoretically
1049          * lose wakeups.
1050          */
1051         return ioc->nr_batch_requests == q->nr_batching ||
1052                 (ioc->nr_batch_requests > 0
1053                 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
1054 }
1055 
1056 /*
1057  * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
1058  * will cause the process to be a "batcher" on all queues in the system. This
1059  * is the behaviour we want though - once it gets a wakeup it should be given
1060  * a nice run.
1061  */
1062 static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
1063 {
1064         if (!ioc || ioc_batching(q, ioc))
1065                 return;
1066 
1067         ioc->nr_batch_requests = q->nr_batching;
1068         ioc->last_waited = jiffies;
1069 }
1070 
1071 static void __freed_request(struct request_list *rl, int sync)
1072 {
1073         struct request_queue *q = rl->q;
1074 
1075         if (rl->count[sync] < queue_congestion_off_threshold(q))
1076                 blk_clear_congested(rl, sync);
1077 
1078         if (rl->count[sync] + 1 <= q->nr_requests) {
1079                 if (waitqueue_active(&rl->wait[sync]))
1080                         wake_up(&rl->wait[sync]);
1081 
1082                 blk_clear_rl_full(rl, sync);
1083         }
1084 }
1085 
1086 /*
1087  * A request has just been released.  Account for it, update the full and
1088  * congestion status, wake up any waiters.   Called under q->queue_lock.
1089  */
1090 static void freed_request(struct request_list *rl, bool sync,
1091                 req_flags_t rq_flags)
1092 {
1093         struct request_queue *q = rl->q;
1094 
1095         q->nr_rqs[sync]--;
1096         rl->count[sync]--;
1097         if (rq_flags & RQF_ELVPRIV)
1098                 q->nr_rqs_elvpriv--;
1099 
1100         __freed_request(rl, sync);
1101 
1102         if (unlikely(rl->starved[sync ^ 1]))
1103                 __freed_request(rl, sync ^ 1);
1104 }
1105 
1106 int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
1107 {
1108         struct request_list *rl;
1109         int on_thresh, off_thresh;
1110 
1111         WARN_ON_ONCE(q->mq_ops);
1112 
1113         spin_lock_irq(q->queue_lock);
1114         q->nr_requests = nr;
1115         blk_queue_congestion_threshold(q);
1116         on_thresh = queue_congestion_on_threshold(q);
1117         off_thresh = queue_congestion_off_threshold(q);
1118 
1119         blk_queue_for_each_rl(rl, q) {
1120                 if (rl->count[BLK_RW_SYNC] >= on_thresh)
1121                         blk_set_congested(rl, BLK_RW_SYNC);
1122                 else if (rl->count[BLK_RW_SYNC] < off_thresh)
1123                         blk_clear_congested(rl, BLK_RW_SYNC);
1124 
1125                 if (rl->count[BLK_RW_ASYNC] >= on_thresh)
1126                         blk_set_congested(rl, BLK_RW_ASYNC);
1127                 else if (rl->count[BLK_RW_ASYNC] < off_thresh)
1128                         blk_clear_congested(rl, BLK_RW_ASYNC);
1129 
1130                 if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
1131                         blk_set_rl_full(rl, BLK_RW_SYNC);
1132                 } else {
1133                         blk_clear_rl_full(rl, BLK_RW_SYNC);
1134                         wake_up(&rl->wait[BLK_RW_SYNC]);
1135                 }
1136 
1137                 if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
1138                         blk_set_rl_full(rl, BLK_RW_ASYNC);
1139                 } else {
1140                         blk_clear_rl_full(rl, BLK_RW_ASYNC);
1141                         wake_up(&rl->wait[BLK_RW_ASYNC]);
1142                 }
1143         }
1144 
1145         spin_unlock_irq(q->queue_lock);
1146         return 0;
1147 }
1148 
1149 /**
1150  * __get_request - get a free request
1151  * @rl: request list to allocate from
1152  * @op: operation and flags
1153  * @bio: bio to allocate request for (can be %NULL)
1154  * @gfp_mask: allocation mask
1155  *
1156  * Get a free request from @q.  This function may fail under memory
1157  * pressure or if @q is dead.
1158  *
1159  * Must be called with @q->queue_lock held and,
1160  * Returns ERR_PTR on failure, with @q->queue_lock held.
1161  * Returns request pointer on success, with @q->queue_lock *not held*.
1162  */
1163 static struct request *__get_request(struct request_list *rl, unsigned int op,
1164                 struct bio *bio, gfp_t gfp_mask)
1165 {
1166         struct request_queue *q = rl->q;
1167         struct request *rq;
1168         struct elevator_type *et = q->elevator->type;
1169         struct io_context *ioc = rq_ioc(bio);
1170         struct io_cq *icq = NULL;
1171         const bool is_sync = op_is_sync(op);
1172         int may_queue;
1173         req_flags_t rq_flags = RQF_ALLOCED;
1174 
1175         lockdep_assert_held(q->queue_lock);
1176 
1177         if (unlikely(blk_queue_dying(q)))
1178                 return ERR_PTR(-ENODEV);
1179 
1180         may_queue = elv_may_queue(q, op);
1181         if (may_queue == ELV_MQUEUE_NO)
1182                 goto rq_starved;
1183 
1184         if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
1185                 if (rl->count[is_sync]+1 >= q->nr_requests) {
1186                         /*
1187                          * The queue will fill after this allocation, so set
1188                          * it as full, and mark this process as "batching".
1189                          * This process will be allowed to complete a batch of
1190                          * requests, others will be blocked.
1191                          */
1192                         if (!blk_rl_full(rl, is_sync)) {
1193                                 ioc_set_batching(q, ioc);
1194                                 blk_set_rl_full(rl, is_sync);
1195                         } else {
1196                                 if (may_queue != ELV_MQUEUE_MUST
1197                                                 && !ioc_batching(q, ioc)) {
1198                                         /*
1199                                          * The queue is full and the allocating
1200                                          * process is not a "batcher", and not
1201                                          * exempted by the IO scheduler
1202                                          */
1203                                         return ERR_PTR(-ENOMEM);
1204                                 }
1205                         }
1206                 }
1207                 blk_set_congested(rl, is_sync);
1208         }
1209 
1210         /*
1211          * Only allow batching queuers to allocate up to 50% over the defined
1212          * limit of requests, otherwise we could have thousands of requests
1213          * allocated with any setting of ->nr_requests
1214          */
1215         if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
1216                 return ERR_PTR(-ENOMEM);
1217 
1218         q->nr_rqs[is_sync]++;
1219         rl->count[is_sync]++;
1220         rl->starved[is_sync] = 0;
1221 
1222         /*
1223          * Decide whether the new request will be managed by elevator.  If
1224          * so, mark @rq_flags and increment elvpriv.  Non-zero elvpriv will
1225          * prevent the current elevator from being destroyed until the new
1226          * request is freed.  This guarantees icq's won't be destroyed and
1227          * makes creating new ones safe.
1228          *
1229          * Flush requests do not use the elevator so skip initialization.
1230          * This allows a request to share the flush and elevator data.
1231          *
1232          * Also, lookup icq while holding queue_lock.  If it doesn't exist,
1233          * it will be created after releasing queue_lock.
1234          */
1235         if (!op_is_flush(op) && !blk_queue_bypass(q)) {
1236                 rq_flags |= RQF_ELVPRIV;
1237                 q->nr_rqs_elvpriv++;
1238                 if (et->icq_cache && ioc)
1239                         icq = ioc_lookup_icq(ioc, q);
1240         }
1241 
1242         if (blk_queue_io_stat(q))
1243                 rq_flags |= RQF_IO_STAT;
1244         spin_unlock_irq(q->queue_lock);
1245 
1246         /* allocate and init request */
1247         rq = mempool_alloc(rl->rq_pool, gfp_mask);
1248         if (!rq)
1249                 goto fail_alloc;
1250 
1251         blk_rq_init(q, rq);
1252         blk_rq_set_rl(rq, rl);
1253         rq->cmd_flags = op;
1254         rq->rq_flags = rq_flags;
1255 
1256         /* init elvpriv */
1257         if (rq_flags & RQF_ELVPRIV) {
1258                 if (unlikely(et->icq_cache && !icq)) {
1259                         if (ioc)
1260                                 icq = ioc_create_icq(ioc, q, gfp_mask);
1261                         if (!icq)
1262                                 goto fail_elvpriv;
1263                 }
1264 
1265                 rq->elv.icq = icq;
1266                 if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
1267                         goto fail_elvpriv;
1268 
1269                 /* @rq->elv.icq holds io_context until @rq is freed */
1270                 if (icq)
1271                         get_io_context(icq->ioc);
1272         }
1273 out:
1274         /*
1275          * ioc may be NULL here, and ioc_batching will be false. That's
1276          * OK, if the queue is under the request limit then requests need
1277          * not count toward the nr_batch_requests limit. There will always
1278          * be some limit enforced by BLK_BATCH_TIME.
1279          */
1280         if (ioc_batching(q, ioc))
1281                 ioc->nr_batch_requests--;
1282 
1283         trace_block_getrq(q, bio, op);
1284         return rq;
1285 
1286 fail_elvpriv:
1287         /*
1288          * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
1289          * and may fail indefinitely under memory pressure and thus
1290          * shouldn't stall IO.  Treat this request as !elvpriv.  This will
1291          * disturb iosched and blkcg but weird is bettern than dead.
1292          */
1293         printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
1294                            __func__, dev_name(q->backing_dev_info->dev));
1295 
1296         rq->rq_flags &= ~RQF_ELVPRIV;
1297         rq->elv.icq = NULL;
1298 
1299         spin_lock_irq(q->queue_lock);
1300         q->nr_rqs_elvpriv--;
1301         spin_unlock_irq(q->queue_lock);
1302         goto out;
1303 
1304 fail_alloc:
1305         /*
1306          * Allocation failed presumably due to memory. Undo anything we
1307          * might have messed up.
1308          *
1309          * Allocating task should really be put onto the front of the wait
1310          * queue, but this is pretty rare.
1311          */
1312         spin_lock_irq(q->queue_lock);
1313         freed_request(rl, is_sync, rq_flags);
1314 
1315         /*
1316          * in the very unlikely event that allocation failed and no
1317          * requests for this direction was pending, mark us starved so that
1318          * freeing of a request in the other direction will notice
1319          * us. another possible fix would be to split the rq mempool into
1320          * READ and WRITE
1321          */
1322 rq_starved:
1323         if (unlikely(rl->count[is_sync] == 0))
1324                 rl->starved[is_sync] = 1;
1325         return ERR_PTR(-ENOMEM);
1326 }
1327 
1328 /**
1329  * get_request - get a free request
1330  * @q: request_queue to allocate request from
1331  * @op: operation and flags
1332  * @bio: bio to allocate request for (can be %NULL)
1333  * @gfp_mask: allocation mask
1334  *
1335  * Get a free request from @q.  If %__GFP_DIRECT_RECLAIM is set in @gfp_mask,
1336  * this function keeps retrying under memory pressure and fails iff @q is dead.
1337  *
1338  * Must be called with @q->queue_lock held and,
1339  * Returns ERR_PTR on failure, with @q->queue_lock held.
1340  * Returns request pointer on success, with @q->queue_lock *not held*.
1341  */
1342 static struct request *get_request(struct request_queue *q, unsigned int op,
1343                 struct bio *bio, gfp_t gfp_mask)
1344 {
1345         const bool is_sync = op_is_sync(op);
1346         DEFINE_WAIT(wait);
1347         struct request_list *rl;
1348         struct request *rq;
1349 
1350         lockdep_assert_held(q->queue_lock);
1351         WARN_ON_ONCE(q->mq_ops);
1352 
1353         rl = blk_get_rl(q, bio);        /* transferred to @rq on success */
1354 retry:
1355         rq = __get_request(rl, op, bio, gfp_mask);
1356         if (!IS_ERR(rq))
1357                 return rq;
1358 
1359         if (op & REQ_NOWAIT) {
1360                 blk_put_rl(rl);
1361                 return ERR_PTR(-EAGAIN);
1362         }
1363 
1364         if (!gfpflags_allow_blocking(gfp_mask) || unlikely(blk_queue_dying(q))) {
1365                 blk_put_rl(rl);
1366                 return rq;
1367         }
1368 
1369         /* wait on @rl and retry */
1370         prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1371                                   TASK_UNINTERRUPTIBLE);
1372 
1373         trace_block_sleeprq(q, bio, op);
1374 
1375         spin_unlock_irq(q->queue_lock);
1376         io_schedule();
1377 
1378         /*
1379          * After sleeping, we become a "batching" process and will be able
1380          * to allocate at least one request, and up to a big batch of them
1381          * for a small period time.  See ioc_batching, ioc_set_batching
1382          */
1383         ioc_set_batching(q, current->io_context);
1384 
1385         spin_lock_irq(q->queue_lock);
1386         finish_wait(&rl->wait[is_sync], &wait);
1387 
1388         goto retry;
1389 }
1390 
1391 static struct request *blk_old_get_request(struct request_queue *q,
1392                                            unsigned int op, gfp_t gfp_mask)
1393 {
1394         struct request *rq;
1395 
1396         WARN_ON_ONCE(q->mq_ops);
1397 
1398         /* create ioc upfront */
1399         create_io_context(gfp_mask, q->node);
1400 
1401         spin_lock_irq(q->queue_lock);
1402         rq = get_request(q, op, NULL, gfp_mask);
1403         if (IS_ERR(rq)) {
1404                 spin_unlock_irq(q->queue_lock);
1405                 return rq;
1406         }
1407 
1408         /* q->queue_lock is unlocked at this point */
1409         rq->__data_len = 0;
1410         rq->__sector = (sector_t) -1;
1411         rq->bio = rq->biotail = NULL;
1412         return rq;
1413 }
1414 
1415 struct request *blk_get_request(struct request_queue *q, unsigned int op,
1416                                 gfp_t gfp_mask)
1417 {
1418         struct request *req;
1419 
1420         if (q->mq_ops) {
1421                 req = blk_mq_alloc_request(q, op,
1422                         (gfp_mask & __GFP_DIRECT_RECLAIM) ?
1423                                 0 : BLK_MQ_REQ_NOWAIT);
1424                 if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
1425                         q->mq_ops->initialize_rq_fn(req);
1426         } else {
1427                 req = blk_old_get_request(q, op, gfp_mask);
1428                 if (!IS_ERR(req) && q->initialize_rq_fn)
1429                         q->initialize_rq_fn(req);
1430         }
1431 
1432         return req;
1433 }
1434 EXPORT_SYMBOL(blk_get_request);
1435 
1436 /**
1437  * blk_requeue_request - put a request back on queue
1438  * @q:          request queue where request should be inserted
1439  * @rq:         request to be inserted
1440  *
1441  * Description:
1442  *    Drivers often keep queueing requests until the hardware cannot accept
1443  *    more, when that condition happens we need to put the request back
1444  *    on the queue. Must be called with queue lock held.
1445  */
1446 void blk_requeue_request(struct request_queue *q, struct request *rq)
1447 {
1448         lockdep_assert_held(q->queue_lock);
1449         WARN_ON_ONCE(q->mq_ops);
1450 
1451         blk_delete_timer(rq);
1452         blk_clear_rq_complete(rq);
1453         trace_block_rq_requeue(q, rq);
1454         wbt_requeue(q->rq_wb, &rq->issue_stat);
1455 
1456         if (rq->rq_flags & RQF_QUEUED)
1457                 blk_queue_end_tag(q, rq);
1458 
1459         BUG_ON(blk_queued_rq(rq));
1460 
1461         elv_requeue_request(q, rq);
1462 }
1463 EXPORT_SYMBOL(blk_requeue_request);
1464 
1465 static void add_acct_request(struct request_queue *q, struct request *rq,
1466                              int where)
1467 {
1468         blk_account_io_start(rq, true);
1469         __elv_add_request(q, rq, where);
1470 }
1471 
1472 static void part_round_stats_single(int cpu, struct hd_struct *part,
1473                                     unsigned long now)
1474 {
1475         int inflight;
1476 
1477         if (now == part->stamp)
1478                 return;
1479 
1480         inflight = part_in_flight(part);
1481         if (inflight) {
1482                 __part_stat_add(cpu, part, time_in_queue,
1483                                 inflight * (now - part->stamp));
1484                 __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1485         }
1486         part->stamp = now;
1487 }
1488 
1489 /**
1490  * part_round_stats() - Round off the performance stats on a struct disk_stats.
1491  * @cpu: cpu number for stats access
1492  * @part: target partition
1493  *
1494  * The average IO queue length and utilisation statistics are maintained
1495  * by observing the current state of the queue length and the amount of
1496  * time it has been in this state for.
1497  *
1498  * Normally, that accounting is done on IO completion, but that can result
1499  * in more than a second's worth of IO being accounted for within any one
1500  * second, leading to >100% utilisation.  To deal with that, we call this
1501  * function to do a round-off before returning the results when reading
1502  * /proc/diskstats.  This accounts immediately for all queue usage up to
1503  * the current jiffies and restarts the counters again.
1504  */
1505 void part_round_stats(int cpu, struct hd_struct *part)
1506 {
1507         unsigned long now = jiffies;
1508 
1509         if (part->partno)
1510                 part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1511         part_round_stats_single(cpu, part, now);
1512 }
1513 EXPORT_SYMBOL_GPL(part_round_stats);
1514 
1515 #ifdef CONFIG_PM
1516 static void blk_pm_put_request(struct request *rq)
1517 {
1518         if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending)
1519                 pm_runtime_mark_last_busy(rq->q->dev);
1520 }
1521 #else
1522 static inline void blk_pm_put_request(struct request *rq) {}
1523 #endif
1524 
1525 void __blk_put_request(struct request_queue *q, struct request *req)
1526 {
1527         req_flags_t rq_flags = req->rq_flags;
1528 
1529         if (unlikely(!q))
1530                 return;
1531 
1532         if (q->mq_ops) {
1533                 blk_mq_free_request(req);
1534                 return;
1535         }
1536 
1537         lockdep_assert_held(q->queue_lock);
1538 
1539         blk_pm_put_request(req);
1540 
1541         elv_completed_request(q, req);
1542 
1543         /* this is a bio leak */
1544         WARN_ON(req->bio != NULL);
1545 
1546         wbt_done(q->rq_wb, &req->issue_stat);
1547 
1548         /*
1549          * Request may not have originated from ll_rw_blk. if not,
1550          * it didn't come out of our reserved rq pools
1551          */
1552         if (rq_flags & RQF_ALLOCED) {
1553                 struct request_list *rl = blk_rq_rl(req);
1554                 bool sync = op_is_sync(req->cmd_flags);
1555 
1556                 BUG_ON(!list_empty(&req->queuelist));
1557                 BUG_ON(ELV_ON_HASH(req));
1558 
1559                 blk_free_request(rl, req);
1560                 freed_request(rl, sync, rq_flags);
1561                 blk_put_rl(rl);
1562         }
1563 }
1564 EXPORT_SYMBOL_GPL(__blk_put_request);
1565 
1566 void blk_put_request(struct request *req)
1567 {
1568         struct request_queue *q = req->q;
1569 
1570         if (q->mq_ops)
1571                 blk_mq_free_request(req);
1572         else {
1573                 unsigned long flags;
1574 
1575                 spin_lock_irqsave(q->queue_lock, flags);
1576                 __blk_put_request(q, req);
1577                 spin_unlock_irqrestore(q->queue_lock, flags);
1578         }
1579 }
1580 EXPORT_SYMBOL(blk_put_request);
1581 
1582 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1583                             struct bio *bio)
1584 {
1585         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1586 
1587         if (!ll_back_merge_fn(q, req, bio))
1588                 return false;
1589 
1590         trace_block_bio_backmerge(q, req, bio);
1591 
1592         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1593                 blk_rq_set_mixed_merge(req);
1594 
1595         req->biotail->bi_next = bio;
1596         req->biotail = bio;
1597         req->__data_len += bio->bi_iter.bi_size;
1598         req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1599 
1600         blk_account_io_start(req, false);
1601         return true;
1602 }
1603 
1604 bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
1605                              struct bio *bio)
1606 {
1607         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1608 
1609         if (!ll_front_merge_fn(q, req, bio))
1610                 return false;
1611 
1612         trace_block_bio_frontmerge(q, req, bio);
1613 
1614         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1615                 blk_rq_set_mixed_merge(req);
1616 
1617         bio->bi_next = req->bio;
1618         req->bio = bio;
1619 
1620         req->__sector = bio->bi_iter.bi_sector;
1621         req->__data_len += bio->bi_iter.bi_size;
1622         req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1623 
1624         blk_account_io_start(req, false);
1625         return true;
1626 }
1627 
1628 bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
1629                 struct bio *bio)
1630 {
1631         unsigned short segments = blk_rq_nr_discard_segments(req);
1632 
1633         if (segments >= queue_max_discard_segments(q))
1634                 goto no_merge;
1635         if (blk_rq_sectors(req) + bio_sectors(bio) >
1636             blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1637                 goto no_merge;
1638 
1639         req->biotail->bi_next = bio;
1640         req->biotail = bio;
1641         req->__data_len += bio->bi_iter.bi_size;
1642         req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1643         req->nr_phys_segments = segments + 1;
1644 
1645         blk_account_io_start(req, false);
1646         return true;
1647 no_merge:
1648         req_set_nomerge(q, req);
1649         return false;
1650 }
1651 
1652 /**
1653  * blk_attempt_plug_merge - try to merge with %current's plugged list
1654  * @q: request_queue new bio is being queued at
1655  * @bio: new bio being queued
1656  * @request_count: out parameter for number of traversed plugged requests
1657  * @same_queue_rq: pointer to &struct request that gets filled in when
1658  * another request associated with @q is found on the plug list
1659  * (optional, may be %NULL)
1660  *
1661  * Determine whether @bio being queued on @q can be merged with a request
1662  * on %current's plugged list.  Returns %true if merge was successful,
1663  * otherwise %false.
1664  *
1665  * Plugging coalesces IOs from the same issuer for the same purpose without
1666  * going through @q->queue_lock.  As such it's more of an issuing mechanism
1667  * than scheduling, and the request, while may have elvpriv data, is not
1668  * added on the elevator at this point.  In addition, we don't have
1669  * reliable access to the elevator outside queue lock.  Only check basic
1670  * merging parameters without querying the elevator.
1671  *
1672  * Caller must ensure !blk_queue_nomerges(q) beforehand.
1673  */
1674 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1675                             unsigned int *request_count,
1676                             struct request **same_queue_rq)
1677 {
1678         struct blk_plug *plug;
1679         struct request *rq;
1680         struct list_head *plug_list;
1681 
1682         plug = current->plug;
1683         if (!plug)
1684                 return false;
1685         *request_count = 0;
1686 
1687         if (q->mq_ops)
1688                 plug_list = &plug->mq_list;
1689         else
1690                 plug_list = &plug->list;
1691 
1692         list_for_each_entry_reverse(rq, plug_list, queuelist) {
1693                 bool merged = false;
1694 
1695                 if (rq->q == q) {
1696                         (*request_count)++;
1697                         /*
1698                          * Only blk-mq multiple hardware queues case checks the
1699                          * rq in the same queue, there should be only one such
1700                          * rq in a queue
1701                          **/
1702                         if (same_queue_rq)
1703                                 *same_queue_rq = rq;
1704                 }
1705 
1706                 if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1707                         continue;
1708 
1709                 switch (blk_try_merge(rq, bio)) {
1710                 case ELEVATOR_BACK_MERGE:
1711                         merged = bio_attempt_back_merge(q, rq, bio);
1712                         break;
1713                 case ELEVATOR_FRONT_MERGE:
1714                         merged = bio_attempt_front_merge(q, rq, bio);
1715                         break;
1716                 case ELEVATOR_DISCARD_MERGE:
1717                         merged = bio_attempt_discard_merge(q, rq, bio);
1718                         break;
1719                 default:
1720                         break;
1721                 }
1722 
1723                 if (merged)
1724                         return true;
1725         }
1726 
1727         return false;
1728 }
1729 
1730 unsigned int blk_plug_queued_count(struct request_queue *q)
1731 {
1732         struct blk_plug *plug;
1733         struct request *rq;
1734         struct list_head *plug_list;
1735         unsigned int ret = 0;
1736 
1737         plug = current->plug;
1738         if (!plug)
1739                 goto out;
1740 
1741         if (q->mq_ops)
1742                 plug_list = &plug->mq_list;
1743         else
1744                 plug_list = &plug->list;
1745 
1746         list_for_each_entry(rq, plug_list, queuelist) {
1747                 if (rq->q == q)
1748                         ret++;
1749         }
1750 out:
1751         return ret;
1752 }
1753 
1754 void blk_init_request_from_bio(struct request *req, struct bio *bio)
1755 {
1756         struct io_context *ioc = rq_ioc(bio);
1757 
1758         if (bio->bi_opf & REQ_RAHEAD)
1759                 req->cmd_flags |= REQ_FAILFAST_MASK;
1760 
1761         req->__sector = bio->bi_iter.bi_sector;
1762         if (ioprio_valid(bio_prio(bio)))
1763                 req->ioprio = bio_prio(bio);
1764         else if (ioc)
1765                 req->ioprio = ioc->ioprio;
1766         else
1767                 req->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
1768         req->write_hint = bio->bi_write_hint;
1769         blk_rq_bio_prep(req->q, req, bio);
1770 }
1771 EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
1772 
1773 static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
1774 {
1775         struct blk_plug *plug;
1776         int where = ELEVATOR_INSERT_SORT;
1777         struct request *req, *free;
1778         unsigned int request_count = 0;
1779         unsigned int wb_acct;
1780 
1781         /*
1782          * low level driver can indicate that it wants pages above a
1783          * certain limit bounced to low memory (ie for highmem, or even
1784          * ISA dma in theory)
1785          */
1786         blk_queue_bounce(q, &bio);
1787 
1788         blk_queue_split(q, &bio);
1789 
1790         if (!bio_integrity_prep(bio))
1791                 return BLK_QC_T_NONE;
1792 
1793         if (op_is_flush(bio->bi_opf)) {
1794                 spin_lock_irq(q->queue_lock);
1795                 where = ELEVATOR_INSERT_FLUSH;
1796                 goto get_rq;
1797         }
1798 
1799         /*
1800          * Check if we can merge with the plugged list before grabbing
1801          * any locks.
1802          */
1803         if (!blk_queue_nomerges(q)) {
1804                 if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
1805                         return BLK_QC_T_NONE;
1806         } else
1807                 request_count = blk_plug_queued_count(q);
1808 
1809         spin_lock_irq(q->queue_lock);
1810 
1811         switch (elv_merge(q, &req, bio)) {
1812         case ELEVATOR_BACK_MERGE:
1813                 if (!bio_attempt_back_merge(q, req, bio))
1814                         break;
1815                 elv_bio_merged(q, req, bio);
1816                 free = attempt_back_merge(q, req);
1817                 if (free)
1818                         __blk_put_request(q, free);
1819                 else
1820                         elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
1821                 goto out_unlock;
1822         case ELEVATOR_FRONT_MERGE:
1823                 if (!bio_attempt_front_merge(q, req, bio))
1824                         break;
1825                 elv_bio_merged(q, req, bio);
1826                 free = attempt_front_merge(q, req);
1827                 if (free)
1828                         __blk_put_request(q, free);
1829                 else
1830                         elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
1831                 goto out_unlock;
1832         default:
1833                 break;
1834         }
1835 
1836 get_rq:
1837         wb_acct = wbt_wait(q->rq_wb, bio, q->queue_lock);
1838 
1839         /*
1840          * Grab a free request. This is might sleep but can not fail.
1841          * Returns with the queue unlocked.
1842          */
1843         req = get_request(q, bio->bi_opf, bio, GFP_NOIO);
1844         if (IS_ERR(req)) {
1845                 __wbt_done(q->rq_wb, wb_acct);
1846                 if (PTR_ERR(req) == -ENOMEM)
1847                         bio->bi_status = BLK_STS_RESOURCE;
1848                 else
1849                         bio->bi_status = BLK_STS_IOERR;
1850                 bio_endio(bio);
1851                 goto out_unlock;
1852         }
1853 
1854         wbt_track(&req->issue_stat, wb_acct);
1855 
1856         /*
1857          * After dropping the lock and possibly sleeping here, our request
1858          * may now be mergeable after it had proven unmergeable (above).
1859          * We don't worry about that case for efficiency. It won't happen
1860          * often, and the elevators are able to handle it.
1861          */
1862         blk_init_request_from_bio(req, bio);
1863 
1864         if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
1865                 req->cpu = raw_smp_processor_id();
1866 
1867         plug = current->plug;
1868         if (plug) {
1869                 /*
1870                  * If this is the first request added after a plug, fire
1871                  * of a plug trace.
1872                  *
1873                  * @request_count may become stale because of schedule
1874                  * out, so check plug list again.
1875                  */
1876                 if (!request_count || list_empty(&plug->list))
1877                         trace_block_plug(q);
1878                 else {
1879                         struct request *last = list_entry_rq(plug->list.prev);
1880                         if (request_count >= BLK_MAX_REQUEST_COUNT ||
1881                             blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
1882                                 blk_flush_plug_list(plug, false);
1883                                 trace_block_plug(q);
1884                         }
1885                 }
1886                 list_add_tail(&req->queuelist, &plug->list);
1887                 blk_account_io_start(req, true);
1888         } else {
1889                 spin_lock_irq(q->queue_lock);
1890                 add_acct_request(q, req, where);
1891                 __blk_run_queue(q);
1892 out_unlock:
1893                 spin_unlock_irq(q->queue_lock);
1894         }
1895 
1896         return BLK_QC_T_NONE;
1897 }
1898 
1899 /*
1900  * If bio->bi_dev is a partition, remap the location
1901  */
1902 static inline void blk_partition_remap(struct bio *bio)
1903 {
1904         struct block_device *bdev = bio->bi_bdev;
1905 
1906         /*
1907          * Zone reset does not include bi_size so bio_sectors() is always 0.
1908          * Include a test for the reset op code and perform the remap if needed.
1909          */
1910         if (bdev != bdev->bd_contains &&
1911             (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET)) {
1912                 struct hd_struct *p = bdev->bd_part;
1913 
1914                 bio->bi_iter.bi_sector += p->start_sect;
1915                 bio->bi_bdev = bdev->bd_contains;
1916 
1917                 trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio,
1918                                       bdev->bd_dev,
1919                                       bio->bi_iter.bi_sector - p->start_sect);
1920         }
1921 }
1922 
1923 static void handle_bad_sector(struct bio *bio)
1924 {
1925         char b[BDEVNAME_SIZE];
1926 
1927         printk(KERN_INFO "attempt to access beyond end of device\n");
1928         printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
1929                         bdevname(bio->bi_bdev, b),
1930                         bio->bi_opf,
1931                         (unsigned long long)bio_end_sector(bio),
1932                         (long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));
1933 }
1934 
1935 #ifdef CONFIG_FAIL_MAKE_REQUEST
1936 
1937 static DECLARE_FAULT_ATTR(fail_make_request);
1938 
1939 static int __init setup_fail_make_request(char *str)
1940 {
1941         return setup_fault_attr(&fail_make_request, str);
1942 }
1943 __setup("fail_make_request=", setup_fail_make_request);
1944 
1945 static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
1946 {
1947         return part->make_it_fail && should_fail(&fail_make_request, bytes);
1948 }
1949 
1950 static int __init fail_make_request_debugfs(void)
1951 {
1952         struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
1953                                                 NULL, &fail_make_request);
1954 
1955         return PTR_ERR_OR_ZERO(dir);
1956 }
1957 
1958 late_initcall(fail_make_request_debugfs);
1959 
1960 #else /* CONFIG_FAIL_MAKE_REQUEST */
1961 
1962 static inline bool should_fail_request(struct hd_struct *part,
1963                                         unsigned int bytes)
1964 {
1965         return false;
1966 }
1967 
1968 #endif /* CONFIG_FAIL_MAKE_REQUEST */
1969 
1970 /*
1971  * Check whether this bio extends beyond the end of the device.
1972  */
1973 static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1974 {
1975         sector_t maxsector;
1976 
1977         if (!nr_sectors)
1978                 return 0;
1979 
1980         /* Test device or partition size, when known. */
1981         maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
1982         if (maxsector) {
1983                 sector_t sector = bio->bi_iter.bi_sector;
1984 
1985                 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1986                         /*
1987                          * This may well happen - the kernel calls bread()
1988                          * without checking the size of the device, e.g., when
1989                          * mounting a device.
1990                          */
1991                         handle_bad_sector(bio);
1992                         return 1;
1993                 }
1994         }
1995 
1996         return 0;
1997 }
1998 
1999 static noinline_for_stack bool
2000 generic_make_request_checks(struct bio *bio)
2001 {
2002         struct request_queue *q;
2003         int nr_sectors = bio_sectors(bio);
2004         blk_status_t status = BLK_STS_IOERR;
2005         char b[BDEVNAME_SIZE];
2006         struct hd_struct *part;
2007 
2008         might_sleep();
2009 
2010         if (bio_check_eod(bio, nr_sectors))
2011                 goto end_io;
2012 
2013         q = bdev_get_queue(bio->bi_bdev);
2014         if (unlikely(!q)) {
2015                 printk(KERN_ERR
2016                        "generic_make_request: Trying to access "
2017                         "nonexistent block-device %s (%Lu)\n",
2018                         bdevname(bio->bi_bdev, b),
2019                         (long long) bio->bi_iter.bi_sector);
2020                 goto end_io;
2021         }
2022 
2023         /*
2024          * For a REQ_NOWAIT based request, return -EOPNOTSUPP
2025          * if queue is not a request based queue.
2026          */
2027 
2028         if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_rq_based(q))
2029                 goto not_supported;
2030 
2031         part = bio->bi_bdev->bd_part;
2032         if (should_fail_request(part, bio->bi_iter.bi_size) ||
2033             should_fail_request(&part_to_disk(part)->part0,
2034                                 bio->bi_iter.bi_size))
2035                 goto end_io;
2036 
2037         /*
2038          * If this device has partitions, remap block n
2039          * of partition p to block n+start(p) of the disk.
2040          */
2041         blk_partition_remap(bio);
2042 
2043         if (bio_check_eod(bio, nr_sectors))
2044                 goto end_io;
2045 
2046         /*
2047          * Filter flush bio's early so that make_request based
2048          * drivers without flush support don't have to worry
2049          * about them.
2050          */
2051         if (op_is_flush(bio->bi_opf) &&
2052             !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
2053                 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
2054                 if (!nr_sectors) {
2055                         status = BLK_STS_OK;
2056                         goto end_io;
2057                 }
2058         }
2059 
2060         switch (bio_op(bio)) {
2061         case REQ_OP_DISCARD:
2062                 if (!blk_queue_discard(q))
2063                         goto not_supported;
2064                 break;
2065         case REQ_OP_SECURE_ERASE:
2066                 if (!blk_queue_secure_erase(q))
2067                         goto not_supported;
2068                 break;
2069         case REQ_OP_WRITE_SAME:
2070                 if (!bdev_write_same(bio->bi_bdev))
2071                         goto not_supported;
2072                 break;
2073         case REQ_OP_ZONE_REPORT:
2074         case REQ_OP_ZONE_RESET:
2075                 if (!bdev_is_zoned(bio->bi_bdev))
2076                         goto not_supported;
2077                 break;
2078         case REQ_OP_WRITE_ZEROES:
2079                 if (!bdev_write_zeroes_sectors(bio->bi_bdev))
2080                         goto not_supported;
2081                 break;
2082         default:
2083                 break;
2084         }
2085 
2086         /*
2087          * Various block parts want %current->io_context and lazy ioc
2088          * allocation ends up trading a lot of pain for a small amount of
2089          * memory.  Just allocate it upfront.  This may fail and block
2090          * layer knows how to live with it.
2091          */
2092         create_io_context(GFP_ATOMIC, q->node);
2093 
2094         if (!blkcg_bio_issue_check(q, bio))
2095                 return false;
2096 
2097         if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
2098                 trace_block_bio_queue(q, bio);
2099                 /* Now that enqueuing has been traced, we need to trace
2100                  * completion as well.
2101                  */
2102                 bio_set_flag(bio, BIO_TRACE_COMPLETION);
2103         }
2104         return true;
2105 
2106 not_supported:
2107         status = BLK_STS_NOTSUPP;
2108 end_io:
2109         bio->bi_status = status;
2110         bio_endio(bio);
2111         return false;
2112 }
2113 
2114 /**
2115  * generic_make_request - hand a buffer to its device driver for I/O
2116  * @bio:  The bio describing the location in memory and on the device.
2117  *
2118  * generic_make_request() is used to make I/O requests of block
2119  * devices. It is passed a &struct bio, which describes the I/O that needs
2120  * to be done.
2121  *
2122  * generic_make_request() does not return any status.  The
2123  * success/failure status of the request, along with notification of
2124  * completion, is delivered asynchronously through the bio->bi_end_io
2125  * function described (one day) else where.
2126  *
2127  * The caller of generic_make_request must make sure that bi_io_vec
2128  * are set to describe the memory buffer, and that bi_dev and bi_sector are
2129  * set to describe the device address, and the
2130  * bi_end_io and optionally bi_private are set to describe how
2131  * completion notification should be signaled.
2132  *
2133  * generic_make_request and the drivers it calls may use bi_next if this
2134  * bio happens to be merged with someone else, and may resubmit the bio to
2135  * a lower device by calling into generic_make_request recursively, which
2136  * means the bio should NOT be touched after the call to ->make_request_fn.
2137  */
2138 blk_qc_t generic_make_request(struct bio *bio)
2139 {
2140         /*
2141          * bio_list_on_stack[0] contains bios submitted by the current
2142          * make_request_fn.
2143          * bio_list_on_stack[1] contains bios that were submitted before
2144          * the current make_request_fn, but that haven't been processed
2145          * yet.
2146          */
2147         struct bio_list bio_list_on_stack[2];
2148         blk_qc_t ret = BLK_QC_T_NONE;
2149 
2150         if (!generic_make_request_checks(bio))
2151                 goto out;
2152 
2153         /*
2154          * We only want one ->make_request_fn to be active at a time, else
2155          * stack usage with stacked devices could be a problem.  So use
2156          * current->bio_list to keep a list of requests submited by a
2157          * make_request_fn function.  current->bio_list is also used as a
2158          * flag to say if generic_make_request is currently active in this
2159          * task or not.  If it is NULL, then no make_request is active.  If
2160          * it is non-NULL, then a make_request is active, and new requests
2161          * should be added at the tail
2162          */
2163         if (current->bio_list) {
2164                 bio_list_add(&current->bio_list[0], bio);
2165                 goto out;
2166         }
2167 
2168         /* following loop may be a bit non-obvious, and so deserves some
2169          * explanation.
2170          * Before entering the loop, bio->bi_next is NULL (as all callers
2171          * ensure that) so we have a list with a single bio.
2172          * We pretend that we have just taken it off a longer list, so
2173          * we assign bio_list to a pointer to the bio_list_on_stack,
2174          * thus initialising the bio_list of new bios to be
2175          * added.  ->make_request() may indeed add some more bios
2176          * through a recursive call to generic_make_request.  If it
2177          * did, we find a non-NULL value in bio_list and re-enter the loop
2178          * from the top.  In this case we really did just take the bio
2179          * of the top of the list (no pretending) and so remove it from
2180          * bio_list, and call into ->make_request() again.
2181          */
2182         BUG_ON(bio->bi_next);
2183         bio_list_init(&bio_list_on_stack[0]);
2184         current->bio_list = bio_list_on_stack;
2185         do {
2186                 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
2187 
2188                 if (likely(blk_queue_enter(q, bio->bi_opf & REQ_NOWAIT) == 0)) {
2189                         struct bio_list lower, same;
2190 
2191                         /* Create a fresh bio_list for all subordinate requests */
2192                         bio_list_on_stack[1] = bio_list_on_stack[0];
2193                         bio_list_init(&bio_list_on_stack[0]);
2194                         ret = q->make_request_fn(q, bio);
2195 
2196                         blk_queue_exit(q);
2197 
2198                         /* sort new bios into those for a lower level
2199                          * and those for the same level
2200                          */
2201                         bio_list_init(&lower);
2202                         bio_list_init(&same);
2203                         while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
2204                                 if (q == bdev_get_queue(bio->bi_bdev))
2205                                         bio_list_add(&same, bio);
2206                                 else
2207                                         bio_list_add(&lower, bio);
2208                         /* now assemble so we handle the lowest level first */
2209                         bio_list_merge(&bio_list_on_stack[0], &lower);
2210                         bio_list_merge(&bio_list_on_stack[0], &same);
2211                         bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
2212                 } else {
2213                         if (unlikely(!blk_queue_dying(q) &&
2214                                         (bio->bi_opf & REQ_NOWAIT)))
2215                                 bio_wouldblock_error(bio);
2216                         else
2217                                 bio_io_error(bio);
2218                 }
2219                 bio = bio_list_pop(&bio_list_on_stack[0]);
2220         } while (bio);
2221         current->bio_list = NULL; /* deactivate */
2222 
2223 out:
2224         return ret;
2225 }
2226 EXPORT_SYMBOL(generic_make_request);
2227 
2228 /**
2229  * submit_bio - submit a bio to the block device layer for I/O
2230  * @bio: The &struct bio which describes the I/O
2231  *
2232  * submit_bio() is very similar in purpose to generic_make_request(), and
2233  * uses that function to do most of the work. Both are fairly rough
2234  * interfaces; @bio must be presetup and ready for I/O.
2235  *
2236  */
2237 blk_qc_t submit_bio(struct bio *bio)
2238 {
2239         /*
2240          * If it's a regular read/write or a barrier with data attached,
2241          * go through the normal accounting stuff before submission.
2242          */
2243         if (bio_has_data(bio)) {
2244                 unsigned int count;
2245 
2246                 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
2247                         count = bdev_logical_block_size(bio->bi_bdev) >> 9;
2248                 else
2249                         count = bio_sectors(bio);
2250 
2251                 if (op_is_write(bio_op(bio))) {
2252                         count_vm_events(PGPGOUT, count);
2253                 } else {
2254                         task_io_account_read(bio->bi_iter.bi_size);
2255                         count_vm_events(PGPGIN, count);
2256                 }
2257 
2258                 if (unlikely(block_dump)) {
2259                         char b[BDEVNAME_SIZE];
2260                         printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
2261                         current->comm, task_pid_nr(current),
2262                                 op_is_write(bio_op(bio)) ? "WRITE" : "READ",
2263                                 (unsigned long long)bio->bi_iter.bi_sector,
2264                                 bdevname(bio->bi_bdev, b),
2265                                 count);
2266                 }
2267         }
2268 
2269         return generic_make_request(bio);
2270 }
2271 EXPORT_SYMBOL(submit_bio);
2272 
2273 /**
2274  * blk_cloned_rq_check_limits - Helper function to check a cloned request
2275  *                              for new the queue limits
2276  * @q:  the queue
2277  * @rq: the request being checked
2278  *
2279  * Description:
2280  *    @rq may have been made based on weaker limitations of upper-level queues
2281  *    in request stacking drivers, and it may violate the limitation of @q.
2282  *    Since the block layer and the underlying device driver trust @rq
2283  *    after it is inserted to @q, it should be checked against @q before
2284  *    the insertion using this generic function.
2285  *
2286  *    Request stacking drivers like request-based dm may change the queue
2287  *    limits when retrying requests on other queues. Those requests need
2288  *    to be checked against the new queue limits again during dispatch.
2289  */
2290 static int blk_cloned_rq_check_limits(struct request_queue *q,
2291                                       struct request *rq)
2292 {
2293         if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
2294                 printk(KERN_ERR "%s: over max size limit.\n", __func__);
2295                 return -EIO;
2296         }
2297 
2298         /*
2299          * queue's settings related to segment counting like q->bounce_pfn
2300          * may differ from that of other stacking queues.
2301          * Recalculate it to check the request correctly on this queue's
2302          * limitation.
2303          */
2304         blk_recalc_rq_segments(rq);
2305         if (rq->nr_phys_segments > queue_max_segments(q)) {
2306                 printk(KERN_ERR "%s: over max segments limit.\n", __func__);
2307                 return -EIO;
2308         }
2309 
2310         return 0;
2311 }
2312 
2313 /**
2314  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
2315  * @q:  the queue to submit the request
2316  * @rq: the request being queued
2317  */
2318 blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
2319 {
2320         unsigned long flags;
2321         int where = ELEVATOR_INSERT_BACK;
2322 
2323         if (blk_cloned_rq_check_limits(q, rq))
2324                 return BLK_STS_IOERR;
2325 
2326         if (rq->rq_disk &&
2327             should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
2328                 return BLK_STS_IOERR;
2329 
2330         if (q->mq_ops) {
2331                 if (blk_queue_io_stat(q))
2332                         blk_account_io_start(rq, true);
2333                 /*
2334                  * Since we have a scheduler attached on the top device,
2335                  * bypass a potential scheduler on the bottom device for
2336                  * insert.
2337                  */
2338                 blk_mq_request_bypass_insert(rq);
2339                 return BLK_STS_OK;
2340         }
2341 
2342         spin_lock_irqsave(q->queue_lock, flags);
2343         if (unlikely(blk_queue_dying(q))) {
2344                 spin_unlock_irqrestore(q->queue_lock, flags);
2345                 return BLK_STS_IOERR;
2346         }
2347 
2348         /*
2349          * Submitting request must be dequeued before calling this function
2350          * because it will be linked to another request_queue
2351          */
2352         BUG_ON(blk_queued_rq(rq));
2353 
2354         if (op_is_flush(rq->cmd_flags))
2355                 where = ELEVATOR_INSERT_FLUSH;
2356 
2357         add_acct_request(q, rq, where);
2358         if (where == ELEVATOR_INSERT_FLUSH)
2359                 __blk_run_queue(q);
2360         spin_unlock_irqrestore(q->queue_lock, flags);
2361 
2362         return BLK_STS_OK;
2363 }
2364 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
2365 
2366 /**
2367  * blk_rq_err_bytes - determine number of bytes till the next failure boundary
2368  * @rq: request to examine
2369  *
2370  * Description:
2371  *     A request could be merge of IOs which require different failure
2372  *     handling.  This function determines the number of bytes which
2373  *     can be failed from the beginning of the request without
2374  *     crossing into area which need to be retried further.
2375  *
2376  * Return:
2377  *     The number of bytes to fail.
2378  */
2379 unsigned int blk_rq_err_bytes(const struct request *rq)
2380 {
2381         unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
2382         unsigned int bytes = 0;
2383         struct bio *bio;
2384 
2385         if (!(rq->rq_flags & RQF_MIXED_MERGE))
2386                 return blk_rq_bytes(rq);
2387 
2388         /*
2389          * Currently the only 'mixing' which can happen is between
2390          * different fastfail types.  We can safely fail portions
2391          * which have all the failfast bits that the first one has -
2392          * the ones which are at least as eager to fail as the first
2393          * one.
2394          */
2395         for (bio = rq->bio; bio; bio = bio->bi_next) {
2396                 if ((bio->bi_opf & ff) != ff)
2397                         break;
2398                 bytes += bio->bi_iter.bi_size;
2399         }
2400 
2401         /* this could lead to infinite loop */
2402         BUG_ON(blk_rq_bytes(rq) && !bytes);
2403         return bytes;
2404 }
2405 EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
2406 
2407 void blk_account_io_completion(struct request *req, unsigned int bytes)
2408 {
2409         if (blk_do_io_stat(req)) {
2410                 const int rw = rq_data_dir(req);
2411                 struct hd_struct *part;
2412                 int cpu;
2413 
2414                 cpu = part_stat_lock();
2415                 part = req->part;
2416                 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
2417                 part_stat_unlock();
2418         }
2419 }
2420 
2421 void blk_account_io_done(struct request *req)
2422 {
2423         /*
2424          * Account IO completion.  flush_rq isn't accounted as a
2425          * normal IO on queueing nor completion.  Accounting the
2426          * containing request is enough.
2427          */
2428         if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
2429                 unsigned long duration = jiffies - req->start_time;
2430                 const int rw = rq_data_dir(req);
2431                 struct hd_struct *part;
2432                 int cpu;
2433 
2434                 cpu = part_stat_lock();
2435                 part = req->part;
2436 
2437                 part_stat_inc(cpu, part, ios[rw]);
2438                 part_stat_add(cpu, part, ticks[rw], duration);
2439                 part_round_stats(cpu, part);
2440                 part_dec_in_flight(part, rw);
2441 
2442                 hd_struct_put(part);
2443                 part_stat_unlock();
2444         }
2445 }
2446 
2447 #ifdef CONFIG_PM
2448 /*
2449  * Don't process normal requests when queue is suspended
2450  * or in the process of suspending/resuming
2451  */
2452 static struct request *blk_pm_peek_request(struct request_queue *q,
2453                                            struct request *rq)
2454 {
2455         if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
2456             (q->rpm_status != RPM_ACTIVE && !(rq->rq_flags & RQF_PM))))
2457                 return NULL;
2458         else
2459                 return rq;
2460 }
2461 #else
2462 static inline struct request *blk_pm_peek_request(struct request_queue *q,
2463                                                   struct request *rq)
2464 {
2465         return rq;
2466 }
2467 #endif
2468 
2469 void blk_account_io_start(struct request *rq, bool new_io)
2470 {
2471         struct hd_struct *part;
2472         int rw = rq_data_dir(rq);
2473         int cpu;
2474 
2475         if (!blk_do_io_stat(rq))
2476                 return;
2477 
2478         cpu = part_stat_lock();
2479 
2480         if (!new_io) {
2481                 part = rq->part;
2482                 part_stat_inc(cpu, part, merges[rw]);
2483         } else {
2484                 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
2485                 if (!hd_struct_try_get(part)) {
2486                         /*
2487                          * The partition is already being removed,
2488                          * the request will be accounted on the disk only
2489                          *
2490                          * We take a reference on disk->part0 although that
2491                          * partition will never be deleted, so we can treat
2492                          * it as any other partition.
2493                          */
2494                         part = &rq->rq_disk->part0;
2495                         hd_struct_get(part);
2496                 }
2497                 part_round_stats(cpu, part);
2498                 part_inc_in_flight(part, rw);
2499                 rq->part = part;
2500         }
2501 
2502         part_stat_unlock();
2503 }
2504 
2505 /**
2506  * blk_peek_request - peek at the top of a request queue
2507  * @q: request queue to peek at
2508  *
2509  * Description:
2510  *     Return the request at the top of @q.  The returned request
2511  *     should be started using blk_start_request() before LLD starts
2512  *     processing it.
2513  *
2514  * Return:
2515  *     Pointer to the request at the top of @q if available.  Null
2516  *     otherwise.
2517  */
2518 struct request *blk_peek_request(struct request_queue *q)
2519 {
2520         struct request *rq;
2521         int ret;
2522 
2523         lockdep_assert_held(q->queue_lock);
2524         WARN_ON_ONCE(q->mq_ops);
2525 
2526         while ((rq = __elv_next_request(q)) != NULL) {
2527 
2528                 rq = blk_pm_peek_request(q, rq);
2529                 if (!rq)
2530                         break;
2531 
2532                 if (!(rq->rq_flags & RQF_STARTED)) {
2533                         /*
2534                          * This is the first time the device driver
2535                          * sees this request (possibly after
2536                          * requeueing).  Notify IO scheduler.
2537                          */
2538                         if (rq->rq_flags & RQF_SORTED)
2539                                 elv_activate_rq(q, rq);
2540 
2541                         /*
2542                          * just mark as started even if we don't start
2543                          * it, a request that has been delayed should
2544                          * not be passed by new incoming requests
2545                          */
2546                         rq->rq_flags |= RQF_STARTED;
2547                         trace_block_rq_issue(q, rq);
2548                 }
2549 
2550                 if (!q->boundary_rq || q->boundary_rq == rq) {
2551                         q->end_sector = rq_end_sector(rq);
2552                         q->boundary_rq = NULL;
2553                 }
2554 
2555                 if (rq->rq_flags & RQF_DONTPREP)
2556                         break;
2557 
2558                 if (q->dma_drain_size && blk_rq_bytes(rq)) {
2559                         /*
2560                          * make sure space for the drain appears we
2561                          * know we can do this because max_hw_segments
2562                          * has been adjusted to be one fewer than the
2563                          * device can handle
2564                          */
2565                         rq->nr_phys_segments++;
2566                 }
2567 
2568                 if (!q->prep_rq_fn)
2569                         break;
2570 
2571                 ret = q->prep_rq_fn(q, rq);
2572                 if (ret == BLKPREP_OK) {
2573                         break;
2574                 } else if (ret == BLKPREP_DEFER) {
2575                         /*
2576                          * the request may have been (partially) prepped.
2577                          * we need to keep this request in the front to
2578                          * avoid resource deadlock.  RQF_STARTED will
2579                          * prevent other fs requests from passing this one.
2580                          */
2581                         if (q->dma_drain_size && blk_rq_bytes(rq) &&
2582                             !(rq->rq_flags & RQF_DONTPREP)) {
2583                                 /*
2584                                  * remove the space for the drain we added
2585                                  * so that we don't add it again
2586                                  */
2587                                 --rq->nr_phys_segments;
2588                         }
2589 
2590                         rq = NULL;
2591                         break;
2592                 } else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
2593                         rq->rq_flags |= RQF_QUIET;
2594                         /*
2595                          * Mark this request as started so we don't trigger
2596                          * any debug logic in the end I/O path.
2597                          */
2598                         blk_start_request(rq);
2599                         __blk_end_request_all(rq, ret == BLKPREP_INVALID ?
2600                                         BLK_STS_TARGET : BLK_STS_IOERR);
2601                 } else {
2602                         printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2603                         break;
2604                 }
2605         }
2606 
2607         return rq;
2608 }
2609 EXPORT_SYMBOL(blk_peek_request);
2610 
2611 void blk_dequeue_request(struct request *rq)
2612 {
2613         struct request_queue *q = rq->q;
2614 
2615         BUG_ON(list_empty(&rq->queuelist));
2616         BUG_ON(ELV_ON_HASH(rq));
2617 
2618         list_del_init(&rq->queuelist);
2619 
2620         /*
2621          * the time frame between a request being removed from the lists
2622          * and to it is freed is accounted as io that is in progress at
2623          * the driver side.
2624          */
2625         if (blk_account_rq(rq)) {
2626                 q->in_flight[rq_is_sync(rq)]++;
2627                 set_io_start_time_ns(rq);
2628         }
2629 }
2630 
2631 /**
2632  * blk_start_request - start request processing on the driver
2633  * @req: request to dequeue
2634  *
2635  * Description:
2636  *     Dequeue @req and start timeout timer on it.  This hands off the
2637  *     request to the driver.
2638  *
2639  *     Block internal functions which don't want to start timer should
2640  *     call blk_dequeue_request().
2641  */
2642 void blk_start_request(struct request *req)
2643 {
2644         lockdep_assert_held(req->q->queue_lock);
2645         WARN_ON_ONCE(req->q->mq_ops);
2646 
2647         blk_dequeue_request(req);
2648 
2649         if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2650                 blk_stat_set_issue(&req->issue_stat, blk_rq_sectors(req));
2651                 req->rq_flags |= RQF_STATS;
2652                 wbt_issue(req->q->rq_wb, &req->issue_stat);
2653         }
2654 
2655         BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));
2656         blk_add_timer(req);
2657 }
2658 EXPORT_SYMBOL(blk_start_request);
2659 
2660 /**
2661  * blk_fetch_request - fetch a request from a request queue
2662  * @q: request queue to fetch a request from
2663  *
2664  * Description:
2665  *     Return the request at the top of @q.  The request is started on
2666  *     return and LLD can start processing it immediately.
2667  *
2668  * Return:
2669  *     Pointer to the request at the top of @q if available.  Null
2670  *     otherwise.
2671  */
2672 struct request *blk_fetch_request(struct request_queue *q)
2673 {
2674         struct request *rq;
2675 
2676         lockdep_assert_held(q->queue_lock);
2677         WARN_ON_ONCE(q->mq_ops);
2678 
2679         rq = blk_peek_request(q);
2680         if (rq)
2681                 blk_start_request(rq);
2682         return rq;
2683 }
2684 EXPORT_SYMBOL(blk_fetch_request);
2685 
2686 /**
2687  * blk_update_request - Special helper function for request stacking drivers
2688  * @req:      the request being processed
2689  * @error:    block status code
2690  * @nr_bytes: number of bytes to complete @req
2691  *
2692  * Description:
2693  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
2694  *     the request structure even if @req doesn't have leftover.
2695  *     If @req has leftover, sets it up for the next range of segments.
2696  *
2697  *     This special helper function is only for request stacking drivers
2698  *     (e.g. request-based dm) so that they can handle partial completion.
2699  *     Actual device drivers should use blk_end_request instead.
2700  *
2701  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
2702  *     %false return from this function.
2703  *
2704  * Return:
2705  *     %false - this request doesn't have any more data
2706  *     %true  - this request has more data
2707  **/
2708 bool blk_update_request(struct request *req, blk_status_t error,
2709                 unsigned int nr_bytes)
2710 {
2711         int total_bytes;
2712 
2713         trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
2714 
2715         if (!req->bio)
2716                 return false;
2717 
2718         if (unlikely(error && !blk_rq_is_passthrough(req) &&
2719                      !(req->rq_flags & RQF_QUIET)))
2720                 print_req_error(req, error);
2721 
2722         blk_account_io_completion(req, nr_bytes);
2723 
2724         total_bytes = 0;
2725         while (req->bio) {
2726                 struct bio *bio = req->bio;
2727                 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
2728 
2729                 if (bio_bytes == bio->bi_iter.bi_size)
2730                         req->bio = bio->bi_next;
2731 
2732                 /* Completion has already been traced */
2733                 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
2734                 req_bio_endio(req, bio, bio_bytes, error);
2735 
2736                 total_bytes += bio_bytes;
2737                 nr_bytes -= bio_bytes;
2738 
2739                 if (!nr_bytes)
2740                         break;
2741         }
2742 
2743         /*
2744          * completely done
2745          */
2746         if (!req->bio) {
2747                 /*
2748                  * Reset counters so that the request stacking driver
2749                  * can find how many bytes remain in the request
2750                  * later.
2751                  */
2752                 req->__data_len = 0;
2753                 return false;
2754         }
2755 
2756         req->__data_len -= total_bytes;
2757 
2758         /* update sector only for requests with clear definition of sector */
2759         if (!blk_rq_is_passthrough(req))
2760                 req->__sector += total_bytes >> 9;
2761 
2762         /* mixed attributes always follow the first bio */
2763         if (req->rq_flags & RQF_MIXED_MERGE) {
2764                 req->cmd_flags &= ~REQ_FAILFAST_MASK;
2765                 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
2766         }
2767 
2768         if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
2769                 /*
2770                  * If total number of sectors is less than the first segment
2771                  * size, something has gone terribly wrong.
2772                  */
2773                 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2774                         blk_dump_rq_flags(req, "request botched");
2775                         req->__data_len = blk_rq_cur_bytes(req);
2776                 }
2777 
2778                 /* recalculate the number of segments */
2779                 blk_recalc_rq_segments(req);
2780         }
2781 
2782         return true;
2783 }
2784 EXPORT_SYMBOL_GPL(blk_update_request);
2785 
2786 static bool blk_update_bidi_request(struct request *rq, blk_status_t error,
2787                                     unsigned int nr_bytes,
2788                                     unsigned int bidi_bytes)
2789 {
2790         if (blk_update_request(rq, error, nr_bytes))
2791                 return true;
2792 
2793         /* Bidi request must be completed as a whole */
2794         if (unlikely(blk_bidi_rq(rq)) &&
2795             blk_update_request(rq->next_rq, error, bidi_bytes))
2796                 return true;
2797 
2798         if (blk_queue_add_random(rq->q))
2799                 add_disk_randomness(rq->rq_disk);
2800 
2801         return false;
2802 }
2803 
2804 /**
2805  * blk_unprep_request - unprepare a request
2806  * @req:        the request
2807  *
2808  * This function makes a request ready for complete resubmission (or
2809  * completion).  It happens only after all error handling is complete,
2810  * so represents the appropriate moment to deallocate any resources
2811  * that were allocated to the request in the prep_rq_fn.  The queue
2812  * lock is held when calling this.
2813  */
2814 void blk_unprep_request(struct request *req)
2815 {
2816         struct request_queue *q = req->q;
2817 
2818         req->rq_flags &= ~RQF_DONTPREP;
2819         if (q->unprep_rq_fn)
2820                 q->unprep_rq_fn(q, req);
2821 }
2822 EXPORT_SYMBOL_GPL(blk_unprep_request);
2823 
2824 void blk_finish_request(struct request *req, blk_status_t error)
2825 {
2826         struct request_queue *q = req->q;
2827 
2828         lockdep_assert_held(req->q->queue_lock);
2829         WARN_ON_ONCE(q->mq_ops);
2830 
2831         if (req->rq_flags & RQF_STATS)
2832                 blk_stat_add(req);
2833 
2834         if (req->rq_flags & RQF_QUEUED)
2835                 blk_queue_end_tag(q, req);
2836 
2837         BUG_ON(blk_queued_rq(req));
2838 
2839         if (unlikely(laptop_mode) && !blk_rq_is_passthrough(req))
2840                 laptop_io_completion(req->q->backing_dev_info);
2841 
2842         blk_delete_timer(req);
2843 
2844         if (req->rq_flags & RQF_DONTPREP)
2845                 blk_unprep_request(req);
2846 
2847         blk_account_io_done(req);
2848 
2849         if (req->end_io) {
2850                 wbt_done(req->q->rq_wb, &req->issue_stat);
2851                 req->end_io(req, error);
2852         } else {
2853                 if (blk_bidi_rq(req))
2854                         __blk_put_request(req->next_rq->q, req->next_rq);
2855 
2856                 __blk_put_request(q, req);
2857         }
2858 }
2859 EXPORT_SYMBOL(blk_finish_request);
2860 
2861 /**
2862  * blk_end_bidi_request - Complete a bidi request
2863  * @rq:         the request to complete
2864  * @error:      block status code
2865  * @nr_bytes:   number of bytes to complete @rq
2866  * @bidi_bytes: number of bytes to complete @rq->next_rq
2867  *
2868  * Description:
2869  *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
2870  *     Drivers that supports bidi can safely call this member for any
2871  *     type of request, bidi or uni.  In the later case @bidi_bytes is
2872  *     just ignored.
2873  *
2874  * Return:
2875  *     %false - we are done with this request
2876  *     %true  - still buffers pending for this request
2877  **/
2878 static bool blk_end_bidi_request(struct request *rq, blk_status_t error,
2879                                  unsigned int nr_bytes, unsigned int bidi_bytes)
2880 {
2881         struct request_queue *q = rq->q;
2882         unsigned long flags;
2883 
2884         WARN_ON_ONCE(q->mq_ops);
2885 
2886         if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2887                 return true;
2888 
2889         spin_lock_irqsave(q->queue_lock, flags);
2890         blk_finish_request(rq, error);
2891         spin_unlock_irqrestore(q->queue_lock, flags);
2892 
2893         return false;
2894 }
2895 
2896 /**
2897  * __blk_end_bidi_request - Complete a bidi request with queue lock held
2898  * @rq:         the request to complete
2899  * @error:      block status code
2900  * @nr_bytes:   number of bytes to complete @rq
2901  * @bidi_bytes: number of bytes to complete @rq->next_rq
2902  *
2903  * Description:
2904  *     Identical to blk_end_bidi_request() except that queue lock is
2905  *     assumed to be locked on entry and remains so on return.
2906  *
2907  * Return:
2908  *     %false - we are done with this request
2909  *     %true  - still buffers pending for this request
2910  **/
2911 static bool __blk_end_bidi_request(struct request *rq, blk_status_t error,
2912                                    unsigned int nr_bytes, unsigned int bidi_bytes)
2913 {
2914         lockdep_assert_held(rq->q->queue_lock);
2915         WARN_ON_ONCE(rq->q->mq_ops);
2916 
2917         if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2918                 return true;
2919 
2920         blk_finish_request(rq, error);
2921 
2922         return false;
2923 }
2924 
2925 /**
2926  * blk_end_request - Helper function for drivers to complete the request.
2927  * @rq:       the request being processed
2928  * @error:    block status code
2929  * @nr_bytes: number of bytes to complete
2930  *
2931  * Description:
2932  *     Ends I/O on a number of bytes attached to @rq.
2933  *     If @rq has leftover, sets it up for the next range of segments.
2934  *
2935  * Return:
2936  *     %false - we are done with this request
2937  *     %true  - still buffers pending for this request
2938  **/
2939 bool blk_end_request(struct request *rq, blk_status_t error,
2940                 unsigned int nr_bytes)
2941 {
2942         WARN_ON_ONCE(rq->q->mq_ops);
2943         return blk_end_bidi_request(rq, error, nr_bytes, 0);
2944 }
2945 EXPORT_SYMBOL(blk_end_request);
2946 
2947 /**
2948  * blk_end_request_all - Helper function for drives to finish the request.
2949  * @rq: the request to finish
2950  * @error: block status code
2951  *
2952  * Description:
2953  *     Completely finish @rq.
2954  */
2955 void blk_end_request_all(struct request *rq, blk_status_t error)
2956 {
2957         bool pending;
2958         unsigned int bidi_bytes = 0;
2959 
2960         if (unlikely(blk_bidi_rq(rq)))
2961                 bidi_bytes = blk_rq_bytes(rq->next_rq);
2962 
2963         pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2964         BUG_ON(pending);
2965 }
2966 EXPORT_SYMBOL(blk_end_request_all);
2967 
2968 /**
2969  * __blk_end_request - Helper function for drivers to complete the request.
2970  * @rq:       the request being processed
2971  * @error:    block status code
2972  * @nr_bytes: number of bytes to complete
2973  *
2974  * Description:
2975  *     Must be called with queue lock held unlike blk_end_request().
2976  *
2977  * Return:
2978  *     %false - we are done with this request
2979  *     %true  - still buffers pending for this request
2980  **/
2981 bool __blk_end_request(struct request *rq, blk_status_t error,
2982                 unsigned int nr_bytes)
2983 {
2984         lockdep_assert_held(rq->q->queue_lock);
2985         WARN_ON_ONCE(rq->q->mq_ops);
2986 
2987         return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2988 }
2989 EXPORT_SYMBOL(__blk_end_request);
2990 
2991 /**
2992  * __blk_end_request_all - Helper function for drives to finish the request.
2993  * @rq: the request to finish
2994  * @error:    block status code
2995  *
2996  * Description:
2997  *     Completely finish @rq.  Must be called with queue lock held.
2998  */
2999 void __blk_end_request_all(struct request *rq, blk_status_t error)
3000 {
3001         bool pending;
3002         unsigned int bidi_bytes = 0;
3003 
3004         lockdep_assert_held(rq->q->queue_lock);
3005         WARN_ON_ONCE(rq->q->mq_ops);
3006 
3007         if (unlikely(blk_bidi_rq(rq)))
3008                 bidi_bytes = blk_rq_bytes(rq->next_rq);
3009 
3010         pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
3011         BUG_ON(pending);
3012 }
3013 EXPORT_SYMBOL(__blk_end_request_all);
3014 
3015 /**
3016  * __blk_end_request_cur - Helper function to finish the current request chunk.
3017  * @rq: the request to finish the current chunk for
3018  * @error:    block status code
3019  *
3020  * Description:
3021  *     Complete the current consecutively mapped chunk from @rq.  Must
3022  *     be called with queue lock held.
3023  *
3024  * Return:
3025  *     %false - we are done with this request
3026  *     %true  - still buffers pending for this request
3027  */
3028 bool __blk_end_request_cur(struct request *rq, blk_status_t error)
3029 {
3030         return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
3031 }
3032 EXPORT_SYMBOL(__blk_end_request_cur);
3033 
3034 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
3035                      struct bio *bio)
3036 {
3037         if (bio_has_data(bio))
3038                 rq->nr_phys_segments = bio_phys_segments(q, bio);
3039 
3040         rq->__data_len = bio->bi_iter.bi_size;
3041         rq->bio = rq->biotail = bio;
3042 
3043         if (bio->bi_bdev)
3044                 rq->rq_disk = bio->bi_bdev->bd_disk;
3045 }
3046 
3047 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
3048 /**
3049  * rq_flush_dcache_pages - Helper function to flush all pages in a request
3050  * @rq: the request to be flushed
3051  *
3052  * Description:
3053  *     Flush all pages in @rq.
3054  */
3055 void rq_flush_dcache_pages(struct request *rq)
3056 {
3057         struct req_iterator iter;
3058         struct bio_vec bvec;
3059 
3060         rq_for_each_segment(bvec, rq, iter)
3061                 flush_dcache_page(bvec.bv_page);
3062 }
3063 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
3064 #endif
3065 
3066 /**
3067  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
3068  * @q : the queue of the device being checked
3069  *
3070  * Description:
3071  *    Check if underlying low-level drivers of a device are busy.
3072  *    If the drivers want to export their busy state, they must set own
3073  *    exporting function using blk_queue_lld_busy() first.
3074  *
3075  *    Basically, this function is used only by request stacking drivers
3076  *    to stop dispatching requests to underlying devices when underlying
3077  *    devices are busy.  This behavior helps more I/O merging on the queue
3078  *    of the request stacking driver and prevents I/O throughput regression
3079  *    on burst I/O load.
3080  *
3081  * Return:
3082  *    0 - Not busy (The request stacking driver should dispatch request)
3083  *    1 - Busy (The request stacking driver should stop dispatching request)
3084  */
3085 int blk_lld_busy(struct request_queue *q)
3086 {
3087         if (q->lld_busy_fn)
3088                 return q->lld_busy_fn(q);
3089 
3090         return 0;
3091 }
3092 EXPORT_SYMBOL_GPL(blk_lld_busy);
3093 
3094 /**
3095  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
3096  * @rq: the clone request to be cleaned up
3097  *
3098  * Description:
3099  *     Free all bios in @rq for a cloned request.
3100  */
3101 void blk_rq_unprep_clone(struct request *rq)
3102 {
3103         struct bio *bio;
3104 
3105         while ((bio = rq->bio) != NULL) {
3106                 rq->bio = bio->bi_next;
3107 
3108                 bio_put(bio);
3109         }
3110 }
3111 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3112 
3113 /*
3114  * Copy attributes of the original request to the clone request.
3115  * The actual data parts (e.g. ->cmd, ->sense) are not copied.
3116  */
3117 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
3118 {
3119         dst->cpu = src->cpu;
3120         dst->__sector = blk_rq_pos(src);
3121         dst->__data_len = blk_rq_bytes(src);
3122         dst->nr_phys_segments = src->nr_phys_segments;
3123         dst->ioprio = src->ioprio;
3124         dst->extra_len = src->extra_len;
3125 }
3126 
3127 /**
3128  * blk_rq_prep_clone - Helper function to setup clone request
3129  * @rq: the request to be setup
3130  * @rq_src: original request to be cloned
3131  * @bs: bio_set that bios for clone are allocated from
3132  * @gfp_mask: memory allocation mask for bio
3133  * @bio_ctr: setup function to be called for each clone bio.
3134  *           Returns %0 for success, non %0 for failure.
3135  * @data: private data to be passed to @bio_ctr
3136  *
3137  * Description:
3138  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3139  *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
3140  *     are not copied, and copying such parts is the caller's responsibility.
3141  *     Also, pages which the original bios are pointing to are not copied
3142  *     and the cloned bios just point same pages.
3143  *     So cloned bios must be completed before original bios, which means
3144  *     the caller must complete @rq before @rq_src.
3145  */
3146 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
3147                       struct bio_set *bs, gfp_t gfp_mask,
3148                       int (*bio_ctr)(struct bio *, struct bio *, void *),
3149                       void *data)
3150 {
3151         struct bio *bio, *bio_src;
3152 
3153         if (!bs)
3154                 bs = fs_bio_set;
3155 
3156         __rq_for_each_bio(bio_src, rq_src) {
3157                 bio = bio_clone_fast(bio_src, gfp_mask, bs);
3158                 if (!bio)
3159                         goto free_and_out;
3160 
3161                 if (bio_ctr && bio_ctr(bio, bio_src, data))
3162                         goto free_and_out;
3163 
3164                 if (rq->bio) {
3165                         rq->biotail->bi_next = bio;
3166                         rq->biotail = bio;
3167                 } else
3168                         rq->bio = rq->biotail = bio;
3169         }
3170 
3171         __blk_rq_prep_clone(rq, rq_src);
3172 
3173         return 0;
3174 
3175 free_and_out:
3176         if (bio)
3177                 bio_put(bio);
3178         blk_rq_unprep_clone(rq);
3179 
3180         return -ENOMEM;
3181 }
3182 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3183 
3184 int kblockd_schedule_work(struct work_struct *work)
3185 {
3186         return queue_work(kblockd_workqueue, work);
3187 }
3188 EXPORT_SYMBOL(kblockd_schedule_work);
3189 
3190 int kblockd_schedule_work_on(int cpu, struct work_struct *work)
3191 {
3192         return queue_work_on(cpu, kblockd_workqueue, work);
3193 }
3194 EXPORT_SYMBOL(kblockd_schedule_work_on);
3195 
3196 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
3197                                 unsigned long delay)
3198 {
3199         return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
3200 }
3201 EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
3202 
3203 int kblockd_schedule_delayed_work(struct delayed_work *dwork,
3204                                   unsigned long delay)
3205 {
3206         return queue_delayed_work(kblockd_workqueue, dwork, delay);
3207 }
3208 EXPORT_SYMBOL(kblockd_schedule_delayed_work);
3209 
3210 int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork,
3211                                      unsigned long delay)
3212 {
3213         return queue_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
3214 }
3215 EXPORT_SYMBOL(kblockd_schedule_delayed_work_on);
3216 
3217 /**
3218  * blk_start_plug - initialize blk_plug and track it inside the task_struct
3219  * @plug:       The &struct blk_plug that needs to be initialized
3220  *
3221  * Description:
3222  *   Tracking blk_plug inside the task_struct will help with auto-flushing the
3223  *   pending I/O should the task end up blocking between blk_start_plug() and
3224  *   blk_finish_plug(). This is important from a performance perspective, but
3225  *   also ensures that we don't deadlock. For instance, if the task is blocking
3226  *   for a memory allocation, memory reclaim could end up wanting to free a
3227  *   page belonging to that request that is currently residing in our private
3228  *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
3229  *   this kind of deadlock.
3230  */
3231 void blk_start_plug(struct blk_plug *plug)
3232 {
3233         struct task_struct *tsk = current;
3234 
3235         /*
3236          * If this is a nested plug, don't actually assign it.
3237          */
3238         if (tsk->plug)
3239                 return;
3240 
3241         INIT_LIST_HEAD(&plug->list);
3242         INIT_LIST_HEAD(&plug->mq_list);
3243         INIT_LIST_HEAD(&plug->cb_list);
3244         /*
3245          * Store ordering should not be needed here, since a potential
3246          * preempt will imply a full memory barrier
3247          */
3248         tsk->plug = plug;
3249 }
3250 EXPORT_SYMBOL(blk_start_plug);
3251 
3252 static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
3253 {
3254         struct request *rqa = container_of(a, struct request, queuelist);
3255         struct request *rqb = container_of(b, struct request, queuelist);
3256 
3257         return !(rqa->q < rqb->q ||
3258                 (rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
3259 }
3260 
3261 /*
3262  * If 'from_schedule' is true, then postpone the dispatch of requests
3263  * until a safe kblockd context. We due this to avoid accidental big
3264  * additional stack usage in driver dispatch, in places where the originally
3265  * plugger did not intend it.
3266  */
3267 static void queue_unplugged(struct request_queue *q, unsigned int depth,
3268                             bool from_schedule)
3269         __releases(q->queue_lock)
3270 {
3271         lockdep_assert_held(q->queue_lock);
3272 
3273         trace_block_unplug(q, depth, !from_schedule);
3274 
3275         if (from_schedule)
3276                 blk_run_queue_async(q);
3277         else
3278                 __blk_run_queue(q);
3279         spin_unlock(q->queue_lock);
3280 }
3281 
3282 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
3283 {
3284         LIST_HEAD(callbacks);
3285 
3286         while (!list_empty(&plug->cb_list)) {
3287                 list_splice_init(&plug->cb_list, &callbacks);
3288 
3289                 while (!list_empty(&callbacks)) {
3290                         struct blk_plug_cb *cb = list_first_entry(&callbacks,
3291                                                           struct blk_plug_cb,
3292                                                           list);
3293                         list_del(&cb->list);
3294                         cb->callback(cb, from_schedule);
3295                 }
3296         }
3297 }
3298 
3299 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
3300                                       int size)
3301 {
3302         struct blk_plug *plug = current->plug;
3303         struct blk_plug_cb *cb;
3304 
3305         if (!plug)
3306                 return NULL;
3307 
3308         list_for_each_entry(cb, &plug->cb_list, list)
3309                 if (cb->callback == unplug && cb->data == data)
3310                         return cb;
3311 
3312         /* Not currently on the callback list */
3313         BUG_ON(size < sizeof(*cb));
3314         cb = kzalloc(size, GFP_ATOMIC);
3315         if (cb) {
3316                 cb->data = data;
3317                 cb->callback = unplug;
3318                 list_add(&cb->list, &plug->cb_list);
3319         }
3320         return cb;
3321 }
3322 EXPORT_SYMBOL(blk_check_plugged);
3323 
3324 void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3325 {
3326         struct request_queue *q;
3327         unsigned long flags;
3328         struct request *rq;
3329         LIST_HEAD(list);
3330         unsigned int depth;
3331 
3332         flush_plug_callbacks(plug, from_schedule);
3333 
3334         if (!list_empty(&plug->mq_list))
3335                 blk_mq_flush_plug_list(plug, from_schedule);
3336 
3337         if (list_empty(&plug->list))
3338                 return;
3339 
3340         list_splice_init(&plug->list, &list);
3341 
3342         list_sort(NULL, &list, plug_rq_cmp);
3343 
3344         q = NULL;
3345         depth = 0;
3346 
3347         /*
3348          * Save and disable interrupts here, to avoid doing it for every
3349          * queue lock we have to take.
3350          */
3351         local_irq_save(flags);
3352         while (!list_empty(&list)) {
3353                 rq = list_entry_rq(list.next);
3354                 list_del_init(&rq->queuelist);
3355                 BUG_ON(!rq->q);
3356                 if (rq->q != q) {
3357                         /*
3358                          * This drops the queue lock
3359                          */
3360                         if (q)
3361                                 queue_unplugged(q, depth, from_schedule);
3362                         q = rq->q;
3363                         depth = 0;
3364                         spin_lock(q->queue_lock);
3365                 }
3366 
3367                 /*
3368                  * Short-circuit if @q is dead
3369                  */
3370                 if (unlikely(blk_queue_dying(q))) {
3371                         __blk_end_request_all(rq, BLK_STS_IOERR);
3372                         continue;
3373                 }
3374 
3375                 /*
3376                  * rq is already accounted, so use raw insert
3377                  */
3378                 if (op_is_flush(rq->cmd_flags))
3379                         __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3380                 else
3381                         __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3382 
3383                 depth++;
3384         }
3385 
3386         /*
3387          * This drops the queue lock
3388          */
3389         if (q)
3390                 queue_unplugged(q, depth, from_schedule);
3391 
3392         local_irq_restore(flags);
3393 }
3394 
3395 void blk_finish_plug(struct blk_plug *plug)
3396 {
3397         if (plug != current->plug)
3398                 return;
3399         blk_flush_plug_list(plug, false);
3400 
3401         current->plug = NULL;
3402 }
3403 EXPORT_SYMBOL(blk_finish_plug);
3404 
3405 #ifdef CONFIG_PM
3406 /**
3407  * blk_pm_runtime_init - Block layer runtime PM initialization routine
3408  * @q: the queue of the device
3409  * @dev: the device the queue belongs to
3410  *
3411  * Description:
3412  *    Initialize runtime-PM-related fields for @q and start auto suspend for
3413  *    @dev. Drivers that want to take advantage of request-based runtime PM
3414  *    should call this function after @dev has been initialized, and its
3415  *    request queue @q has been allocated, and runtime PM for it can not happen
3416  *    yet(either due to disabled/forbidden or its usage_count > 0). In most
3417  *    cases, driver should call this function before any I/O has taken place.
3418  *
3419  *    This function takes care of setting up using auto suspend for the device,
3420  *    the autosuspend delay is set to -1 to make runtime suspend impossible
3421  *    until an updated value is either set by user or by driver. Drivers do
3422  *    not need to touch other autosuspend settings.
3423  *
3424  *    The block layer runtime PM is request based, so only works for drivers
3425  *    that use request as their IO unit instead of those directly use bio's.
3426  */
3427 void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
3428 {
3429         /* not support for RQF_PM and ->rpm_status in blk-mq yet */
3430         if (q->mq_ops)
3431                 return;
3432 
3433         q->dev = dev;
3434         q->rpm_status = RPM_ACTIVE;
3435         pm_runtime_set_autosuspend_delay(q->dev, -1);
3436         pm_runtime_use_autosuspend(q->dev);
3437 }
3438 EXPORT_SYMBOL(blk_pm_runtime_init);
3439 
3440 /**
3441  * blk_pre_runtime_suspend - Pre runtime suspend check
3442  * @q: the queue of the device
3443  *
3444  * Description:
3445  *    This function will check if runtime suspend is allowed for the device
3446  *    by examining if there are any requests pending in the queue. If there
3447  *    are requests pending, the device can not be runtime suspended; otherwise,
3448  *    the queue's status will be updated to SUSPENDING and the driver can
3449  *    proceed to suspend the device.
3450  *
3451  *    For the not allowed case, we mark last busy for the device so that
3452  *    runtime PM core will try to autosuspend it some time later.
3453  *
3454  *    This function should be called near the start of the device's
3455  *    runtime_suspend callback.
3456  *
3457  * Return:
3458  *    0         - OK to runtime suspend the device
3459  *    -EBUSY    - Device should not be runtime suspended
3460  */
3461 int blk_pre_runtime_suspend(struct request_queue *q)
3462 {
3463         int ret = 0;
3464 
3465         if (!q->dev)
3466                 return ret;
3467 
3468         spin_lock_irq(q->queue_lock);
3469         if (q->nr_pending) {
3470                 ret = -EBUSY;
3471                 pm_runtime_mark_last_busy(q->dev);
3472         } else {
3473                 q->rpm_status = RPM_SUSPENDING;
3474         }
3475         spin_unlock_irq(q->queue_lock);
3476         return ret;
3477 }
3478 EXPORT_SYMBOL(blk_pre_runtime_suspend);
3479 
3480 /**
3481  * blk_post_runtime_suspend - Post runtime suspend processing
3482  * @q: the queue of the device
3483  * @err: return value of the device's runtime_suspend function
3484  *
3485  * Description:
3486  *    Update the queue's runtime status according to the return value of the
3487  *    device's runtime suspend function and mark last busy for the device so
3488  *    that PM core will try to auto suspend the device at a later time.
3489  *
3490  *    This function should be called near the end of the device's
3491  *    runtime_suspend callback.
3492  */
3493 void blk_post_runtime_suspend(struct request_queue *q, int err)
3494 {
3495         if (!q->dev)
3496                 return;
3497 
3498         spin_lock_irq(q->queue_lock);
3499         if (!err) {
3500                 q->rpm_status = RPM_SUSPENDED;
3501         } else {
3502                 q->rpm_status = RPM_ACTIVE;
3503                 pm_runtime_mark_last_busy(q->dev);
3504         }
3505         spin_unlock_irq(q->queue_lock);
3506 }
3507 EXPORT_SYMBOL(blk_post_runtime_suspend);
3508 
3509 /**
3510  * blk_pre_runtime_resume - Pre runtime resume processing
3511  * @q: the queue of the device
3512  *
3513  * Description:
3514  *    Update the queue's runtime status to RESUMING in preparation for the
3515  *    runtime resume of the device.
3516  *
3517  *    This function should be called near the start of the device's
3518  *    runtime_resume callback.
3519  */
3520 void blk_pre_runtime_resume(struct request_queue *q)
3521 {
3522         if (!q->dev)
3523                 return;
3524 
3525         spin_lock_irq(q->queue_lock);
3526         q->rpm_status = RPM_RESUMING;
3527         spin_unlock_irq(q->queue_lock);
3528 }
3529 EXPORT_SYMBOL(blk_pre_runtime_resume);
3530 
3531 /**
3532  * blk_post_runtime_resume - Post runtime resume processing
3533  * @q: the queue of the device
3534  * @err: return value of the device's runtime_resume function
3535  *
3536  * Description:
3537  *    Update the queue's runtime status according to the return value of the
3538  *    device's runtime_resume function. If it is successfully resumed, process
3539  *    the requests that are queued into the device's queue when it is resuming
3540  *    and then mark last busy and initiate autosuspend for it.
3541  *
3542  *    This function should be called near the end of the device's
3543  *    runtime_resume callback.
3544  */
3545 void blk_post_runtime_resume(struct request_queue *q, int err)
3546 {
3547         if (!q->dev)
3548                 return;
3549 
3550         spin_lock_irq(q->queue_lock);
3551         if (!err) {
3552                 q->rpm_status = RPM_ACTIVE;
3553                 __blk_run_queue(q);
3554                 pm_runtime_mark_last_busy(q->dev);
3555                 pm_request_autosuspend(q->dev);
3556         } else {
3557                 q->rpm_status = RPM_SUSPENDED;
3558         }
3559         spin_unlock_irq(q->queue_lock);
3560 }
3561 EXPORT_SYMBOL(blk_post_runtime_resume);
3562 
3563 /**
3564  * blk_set_runtime_active - Force runtime status of the queue to be active
3565  * @q: the queue of the device
3566  *
3567  * If the device is left runtime suspended during system suspend the resume
3568  * hook typically resumes the device and corrects runtime status
3569  * accordingly. However, that does not affect the queue runtime PM status
3570  * which is still "suspended". This prevents processing requests from the
3571  * queue.
3572  *
3573  * This function can be used in driver's resume hook to correct queue
3574  * runtime PM status and re-enable peeking requests from the queue. It
3575  * should be called before first request is added to the queue.
3576  */
3577 void blk_set_runtime_active(struct request_queue *q)
3578 {
3579         spin_lock_irq(q->queue_lock);
3580         q->rpm_status = RPM_ACTIVE;
3581         pm_runtime_mark_last_busy(q->dev);
3582         pm_request_autosuspend(q->dev);
3583         spin_unlock_irq(q->queue_lock);
3584 }
3585 EXPORT_SYMBOL(blk_set_runtime_active);
3586 #endif
3587 
3588 int __init blk_dev_init(void)
3589 {
3590         BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
3591         BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3592                         FIELD_SIZEOF(struct request, cmd_flags));
3593         BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3594                         FIELD_SIZEOF(struct bio, bi_opf));
3595 
3596         /* used for unplugging and affects IO latency/throughput - HIGHPRI */
3597         kblockd_workqueue = alloc_workqueue("kblockd",
3598                                             WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
3599         if (!kblockd_workqueue)
3600                 panic("Failed to create kblockd\n");
3601 
3602         request_cachep = kmem_cache_create("blkdev_requests",
3603                         sizeof(struct request), 0, SLAB_PANIC, NULL);
3604 
3605         blk_requestq_cachep = kmem_cache_create("request_queue",
3606                         sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
3607 
3608 #ifdef CONFIG_DEBUG_FS
3609         blk_debugfs_root = debugfs_create_dir("block", NULL);
3610 #endif
3611 
3612         return 0;
3613 }
3614 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp