~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/block/blk-mq.c

Version: ~ [ linux-6.0-rc6 ] ~ [ linux-5.19.10 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.69 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.144 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.214 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.259 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.294 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.329 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Block multiqueue core code
  4  *
  5  * Copyright (C) 2013-2014 Jens Axboe
  6  * Copyright (C) 2013-2014 Christoph Hellwig
  7  */
  8 #include <linux/kernel.h>
  9 #include <linux/module.h>
 10 #include <linux/backing-dev.h>
 11 #include <linux/bio.h>
 12 #include <linux/blkdev.h>
 13 #include <linux/kmemleak.h>
 14 #include <linux/mm.h>
 15 #include <linux/init.h>
 16 #include <linux/slab.h>
 17 #include <linux/workqueue.h>
 18 #include <linux/smp.h>
 19 #include <linux/llist.h>
 20 #include <linux/list_sort.h>
 21 #include <linux/cpu.h>
 22 #include <linux/cache.h>
 23 #include <linux/sched/sysctl.h>
 24 #include <linux/sched/topology.h>
 25 #include <linux/sched/signal.h>
 26 #include <linux/delay.h>
 27 #include <linux/crash_dump.h>
 28 #include <linux/prefetch.h>
 29 #include <linux/blk-crypto.h>
 30 
 31 #include <trace/events/block.h>
 32 
 33 #include <linux/blk-mq.h>
 34 #include <linux/t10-pi.h>
 35 #include "blk.h"
 36 #include "blk-mq.h"
 37 #include "blk-mq-debugfs.h"
 38 #include "blk-mq-tag.h"
 39 #include "blk-pm.h"
 40 #include "blk-stat.h"
 41 #include "blk-mq-sched.h"
 42 #include "blk-rq-qos.h"
 43 
 44 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
 45 
 46 static void blk_mq_poll_stats_start(struct request_queue *q);
 47 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
 48 
 49 static int blk_mq_poll_stats_bkt(const struct request *rq)
 50 {
 51         int ddir, sectors, bucket;
 52 
 53         ddir = rq_data_dir(rq);
 54         sectors = blk_rq_stats_sectors(rq);
 55 
 56         bucket = ddir + 2 * ilog2(sectors);
 57 
 58         if (bucket < 0)
 59                 return -1;
 60         else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
 61                 return ddir + BLK_MQ_POLL_STATS_BKTS - 2;
 62 
 63         return bucket;
 64 }
 65 
 66 /*
 67  * Check if any of the ctx, dispatch list or elevator
 68  * have pending work in this hardware queue.
 69  */
 70 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
 71 {
 72         return !list_empty_careful(&hctx->dispatch) ||
 73                 sbitmap_any_bit_set(&hctx->ctx_map) ||
 74                         blk_mq_sched_has_work(hctx);
 75 }
 76 
 77 /*
 78  * Mark this ctx as having pending work in this hardware queue
 79  */
 80 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
 81                                      struct blk_mq_ctx *ctx)
 82 {
 83         const int bit = ctx->index_hw[hctx->type];
 84 
 85         if (!sbitmap_test_bit(&hctx->ctx_map, bit))
 86                 sbitmap_set_bit(&hctx->ctx_map, bit);
 87 }
 88 
 89 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
 90                                       struct blk_mq_ctx *ctx)
 91 {
 92         const int bit = ctx->index_hw[hctx->type];
 93 
 94         sbitmap_clear_bit(&hctx->ctx_map, bit);
 95 }
 96 
 97 struct mq_inflight {
 98         struct block_device *part;
 99         unsigned int inflight[2];
100 };
101 
102 static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
103                                   struct request *rq, void *priv,
104                                   bool reserved)
105 {
106         struct mq_inflight *mi = priv;
107 
108         if ((!mi->part->bd_partno || rq->part == mi->part) &&
109             blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
110                 mi->inflight[rq_data_dir(rq)]++;
111 
112         return true;
113 }
114 
115 unsigned int blk_mq_in_flight(struct request_queue *q,
116                 struct block_device *part)
117 {
118         struct mq_inflight mi = { .part = part };
119 
120         blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
121 
122         return mi.inflight[0] + mi.inflight[1];
123 }
124 
125 void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
126                 unsigned int inflight[2])
127 {
128         struct mq_inflight mi = { .part = part };
129 
130         blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
131         inflight[0] = mi.inflight[0];
132         inflight[1] = mi.inflight[1];
133 }
134 
135 void blk_freeze_queue_start(struct request_queue *q)
136 {
137         mutex_lock(&q->mq_freeze_lock);
138         if (++q->mq_freeze_depth == 1) {
139                 percpu_ref_kill(&q->q_usage_counter);
140                 mutex_unlock(&q->mq_freeze_lock);
141                 if (queue_is_mq(q))
142                         blk_mq_run_hw_queues(q, false);
143         } else {
144                 mutex_unlock(&q->mq_freeze_lock);
145         }
146 }
147 EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
148 
149 void blk_mq_freeze_queue_wait(struct request_queue *q)
150 {
151         wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
152 }
153 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
154 
155 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
156                                      unsigned long timeout)
157 {
158         return wait_event_timeout(q->mq_freeze_wq,
159                                         percpu_ref_is_zero(&q->q_usage_counter),
160                                         timeout);
161 }
162 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
163 
164 /*
165  * Guarantee no request is in use, so we can change any data structure of
166  * the queue afterward.
167  */
168 void blk_freeze_queue(struct request_queue *q)
169 {
170         /*
171          * In the !blk_mq case we are only calling this to kill the
172          * q_usage_counter, otherwise this increases the freeze depth
173          * and waits for it to return to zero.  For this reason there is
174          * no blk_unfreeze_queue(), and blk_freeze_queue() is not
175          * exported to drivers as the only user for unfreeze is blk_mq.
176          */
177         blk_freeze_queue_start(q);
178         blk_mq_freeze_queue_wait(q);
179 }
180 
181 void blk_mq_freeze_queue(struct request_queue *q)
182 {
183         /*
184          * ...just an alias to keep freeze and unfreeze actions balanced
185          * in the blk_mq_* namespace
186          */
187         blk_freeze_queue(q);
188 }
189 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
190 
191 void blk_mq_unfreeze_queue(struct request_queue *q)
192 {
193         mutex_lock(&q->mq_freeze_lock);
194         q->mq_freeze_depth--;
195         WARN_ON_ONCE(q->mq_freeze_depth < 0);
196         if (!q->mq_freeze_depth) {
197                 percpu_ref_resurrect(&q->q_usage_counter);
198                 wake_up_all(&q->mq_freeze_wq);
199         }
200         mutex_unlock(&q->mq_freeze_lock);
201 }
202 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
203 
204 /*
205  * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
206  * mpt3sas driver such that this function can be removed.
207  */
208 void blk_mq_quiesce_queue_nowait(struct request_queue *q)
209 {
210         blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
211 }
212 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
213 
214 /**
215  * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
216  * @q: request queue.
217  *
218  * Note: this function does not prevent that the struct request end_io()
219  * callback function is invoked. Once this function is returned, we make
220  * sure no dispatch can happen until the queue is unquiesced via
221  * blk_mq_unquiesce_queue().
222  */
223 void blk_mq_quiesce_queue(struct request_queue *q)
224 {
225         struct blk_mq_hw_ctx *hctx;
226         unsigned int i;
227         bool rcu = false;
228 
229         blk_mq_quiesce_queue_nowait(q);
230 
231         queue_for_each_hw_ctx(q, hctx, i) {
232                 if (hctx->flags & BLK_MQ_F_BLOCKING)
233                         synchronize_srcu(hctx->srcu);
234                 else
235                         rcu = true;
236         }
237         if (rcu)
238                 synchronize_rcu();
239 }
240 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
241 
242 /*
243  * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
244  * @q: request queue.
245  *
246  * This function recovers queue into the state before quiescing
247  * which is done by blk_mq_quiesce_queue.
248  */
249 void blk_mq_unquiesce_queue(struct request_queue *q)
250 {
251         blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
252 
253         /* dispatch requests which are inserted during quiescing */
254         blk_mq_run_hw_queues(q, true);
255 }
256 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
257 
258 void blk_mq_wake_waiters(struct request_queue *q)
259 {
260         struct blk_mq_hw_ctx *hctx;
261         unsigned int i;
262 
263         queue_for_each_hw_ctx(q, hctx, i)
264                 if (blk_mq_hw_queue_mapped(hctx))
265                         blk_mq_tag_wakeup_all(hctx->tags, true);
266 }
267 
268 /*
269  * Only need start/end time stamping if we have iostat or
270  * blk stats enabled, or using an IO scheduler.
271  */
272 static inline bool blk_mq_need_time_stamp(struct request *rq)
273 {
274         return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator;
275 }
276 
277 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
278                 unsigned int tag, u64 alloc_time_ns)
279 {
280         struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
281         struct request *rq = tags->static_rqs[tag];
282 
283         if (data->q->elevator) {
284                 rq->tag = BLK_MQ_NO_TAG;
285                 rq->internal_tag = tag;
286         } else {
287                 rq->tag = tag;
288                 rq->internal_tag = BLK_MQ_NO_TAG;
289         }
290 
291         /* csd/requeue_work/fifo_time is initialized before use */
292         rq->q = data->q;
293         rq->mq_ctx = data->ctx;
294         rq->mq_hctx = data->hctx;
295         rq->rq_flags = 0;
296         rq->cmd_flags = data->cmd_flags;
297         if (data->flags & BLK_MQ_REQ_PM)
298                 rq->rq_flags |= RQF_PM;
299         if (blk_queue_io_stat(data->q))
300                 rq->rq_flags |= RQF_IO_STAT;
301         INIT_LIST_HEAD(&rq->queuelist);
302         INIT_HLIST_NODE(&rq->hash);
303         RB_CLEAR_NODE(&rq->rb_node);
304         rq->rq_disk = NULL;
305         rq->part = NULL;
306 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
307         rq->alloc_time_ns = alloc_time_ns;
308 #endif
309         if (blk_mq_need_time_stamp(rq))
310                 rq->start_time_ns = ktime_get_ns();
311         else
312                 rq->start_time_ns = 0;
313         rq->io_start_time_ns = 0;
314         rq->stats_sectors = 0;
315         rq->nr_phys_segments = 0;
316 #if defined(CONFIG_BLK_DEV_INTEGRITY)
317         rq->nr_integrity_segments = 0;
318 #endif
319         blk_crypto_rq_set_defaults(rq);
320         /* tag was already set */
321         WRITE_ONCE(rq->deadline, 0);
322 
323         rq->timeout = 0;
324 
325         rq->end_io = NULL;
326         rq->end_io_data = NULL;
327 
328         data->ctx->rq_dispatched[op_is_sync(data->cmd_flags)]++;
329         refcount_set(&rq->ref, 1);
330 
331         if (!op_is_flush(data->cmd_flags)) {
332                 struct elevator_queue *e = data->q->elevator;
333 
334                 rq->elv.icq = NULL;
335                 if (e && e->type->ops.prepare_request) {
336                         if (e->type->icq_cache)
337                                 blk_mq_sched_assign_ioc(rq);
338 
339                         e->type->ops.prepare_request(rq);
340                         rq->rq_flags |= RQF_ELVPRIV;
341                 }
342         }
343 
344         data->hctx->queued++;
345         return rq;
346 }
347 
348 static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data)
349 {
350         struct request_queue *q = data->q;
351         struct elevator_queue *e = q->elevator;
352         u64 alloc_time_ns = 0;
353         unsigned int tag;
354 
355         /* alloc_time includes depth and tag waits */
356         if (blk_queue_rq_alloc_time(q))
357                 alloc_time_ns = ktime_get_ns();
358 
359         if (data->cmd_flags & REQ_NOWAIT)
360                 data->flags |= BLK_MQ_REQ_NOWAIT;
361 
362         if (e) {
363                 /*
364                  * Flush/passthrough requests are special and go directly to the
365                  * dispatch list. Don't include reserved tags in the
366                  * limiting, as it isn't useful.
367                  */
368                 if (!op_is_flush(data->cmd_flags) &&
369                     !blk_op_is_passthrough(data->cmd_flags) &&
370                     e->type->ops.limit_depth &&
371                     !(data->flags & BLK_MQ_REQ_RESERVED))
372                         e->type->ops.limit_depth(data->cmd_flags, data);
373         }
374 
375 retry:
376         data->ctx = blk_mq_get_ctx(q);
377         data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
378         if (!e)
379                 blk_mq_tag_busy(data->hctx);
380 
381         /*
382          * Waiting allocations only fail because of an inactive hctx.  In that
383          * case just retry the hctx assignment and tag allocation as CPU hotplug
384          * should have migrated us to an online CPU by now.
385          */
386         tag = blk_mq_get_tag(data);
387         if (tag == BLK_MQ_NO_TAG) {
388                 if (data->flags & BLK_MQ_REQ_NOWAIT)
389                         return NULL;
390 
391                 /*
392                  * Give up the CPU and sleep for a random short time to ensure
393                  * that thread using a realtime scheduling class are migrated
394                  * off the CPU, and thus off the hctx that is going away.
395                  */
396                 msleep(3);
397                 goto retry;
398         }
399         return blk_mq_rq_ctx_init(data, tag, alloc_time_ns);
400 }
401 
402 struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
403                 blk_mq_req_flags_t flags)
404 {
405         struct blk_mq_alloc_data data = {
406                 .q              = q,
407                 .flags          = flags,
408                 .cmd_flags      = op,
409         };
410         struct request *rq;
411         int ret;
412 
413         ret = blk_queue_enter(q, flags);
414         if (ret)
415                 return ERR_PTR(ret);
416 
417         rq = __blk_mq_alloc_request(&data);
418         if (!rq)
419                 goto out_queue_exit;
420         rq->__data_len = 0;
421         rq->__sector = (sector_t) -1;
422         rq->bio = rq->biotail = NULL;
423         return rq;
424 out_queue_exit:
425         blk_queue_exit(q);
426         return ERR_PTR(-EWOULDBLOCK);
427 }
428 EXPORT_SYMBOL(blk_mq_alloc_request);
429 
430 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
431         unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx)
432 {
433         struct blk_mq_alloc_data data = {
434                 .q              = q,
435                 .flags          = flags,
436                 .cmd_flags      = op,
437         };
438         u64 alloc_time_ns = 0;
439         unsigned int cpu;
440         unsigned int tag;
441         int ret;
442 
443         /* alloc_time includes depth and tag waits */
444         if (blk_queue_rq_alloc_time(q))
445                 alloc_time_ns = ktime_get_ns();
446 
447         /*
448          * If the tag allocator sleeps we could get an allocation for a
449          * different hardware context.  No need to complicate the low level
450          * allocator for this for the rare use case of a command tied to
451          * a specific queue.
452          */
453         if (WARN_ON_ONCE(!(flags & (BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED))))
454                 return ERR_PTR(-EINVAL);
455 
456         if (hctx_idx >= q->nr_hw_queues)
457                 return ERR_PTR(-EIO);
458 
459         ret = blk_queue_enter(q, flags);
460         if (ret)
461                 return ERR_PTR(ret);
462 
463         /*
464          * Check if the hardware context is actually mapped to anything.
465          * If not tell the caller that it should skip this queue.
466          */
467         ret = -EXDEV;
468         data.hctx = q->queue_hw_ctx[hctx_idx];
469         if (!blk_mq_hw_queue_mapped(data.hctx))
470                 goto out_queue_exit;
471         cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
472         data.ctx = __blk_mq_get_ctx(q, cpu);
473 
474         if (!q->elevator)
475                 blk_mq_tag_busy(data.hctx);
476 
477         ret = -EWOULDBLOCK;
478         tag = blk_mq_get_tag(&data);
479         if (tag == BLK_MQ_NO_TAG)
480                 goto out_queue_exit;
481         return blk_mq_rq_ctx_init(&data, tag, alloc_time_ns);
482 
483 out_queue_exit:
484         blk_queue_exit(q);
485         return ERR_PTR(ret);
486 }
487 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
488 
489 static void __blk_mq_free_request(struct request *rq)
490 {
491         struct request_queue *q = rq->q;
492         struct blk_mq_ctx *ctx = rq->mq_ctx;
493         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
494         const int sched_tag = rq->internal_tag;
495 
496         blk_crypto_free_request(rq);
497         blk_pm_mark_last_busy(rq);
498         rq->mq_hctx = NULL;
499         if (rq->tag != BLK_MQ_NO_TAG)
500                 blk_mq_put_tag(hctx->tags, ctx, rq->tag);
501         if (sched_tag != BLK_MQ_NO_TAG)
502                 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
503         blk_mq_sched_restart(hctx);
504         blk_queue_exit(q);
505 }
506 
507 void blk_mq_free_request(struct request *rq)
508 {
509         struct request_queue *q = rq->q;
510         struct elevator_queue *e = q->elevator;
511         struct blk_mq_ctx *ctx = rq->mq_ctx;
512         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
513 
514         if (rq->rq_flags & RQF_ELVPRIV) {
515                 if (e && e->type->ops.finish_request)
516                         e->type->ops.finish_request(rq);
517                 if (rq->elv.icq) {
518                         put_io_context(rq->elv.icq->ioc);
519                         rq->elv.icq = NULL;
520                 }
521         }
522 
523         ctx->rq_completed[rq_is_sync(rq)]++;
524         if (rq->rq_flags & RQF_MQ_INFLIGHT)
525                 __blk_mq_dec_active_requests(hctx);
526 
527         if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
528                 laptop_io_completion(q->backing_dev_info);
529 
530         rq_qos_done(q, rq);
531 
532         WRITE_ONCE(rq->state, MQ_RQ_IDLE);
533         if (refcount_dec_and_test(&rq->ref))
534                 __blk_mq_free_request(rq);
535 }
536 EXPORT_SYMBOL_GPL(blk_mq_free_request);
537 
538 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
539 {
540         u64 now = 0;
541 
542         if (blk_mq_need_time_stamp(rq))
543                 now = ktime_get_ns();
544 
545         if (rq->rq_flags & RQF_STATS) {
546                 blk_mq_poll_stats_start(rq->q);
547                 blk_stat_add(rq, now);
548         }
549 
550         blk_mq_sched_completed_request(rq, now);
551 
552         blk_account_io_done(rq, now);
553 
554         if (rq->end_io) {
555                 rq_qos_done(rq->q, rq);
556                 rq->end_io(rq, error);
557         } else {
558                 blk_mq_free_request(rq);
559         }
560 }
561 EXPORT_SYMBOL(__blk_mq_end_request);
562 
563 void blk_mq_end_request(struct request *rq, blk_status_t error)
564 {
565         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
566                 BUG();
567         __blk_mq_end_request(rq, error);
568 }
569 EXPORT_SYMBOL(blk_mq_end_request);
570 
571 static void blk_complete_reqs(struct llist_head *list)
572 {
573         struct llist_node *entry = llist_reverse_order(llist_del_all(list));
574         struct request *rq, *next;
575 
576         llist_for_each_entry_safe(rq, next, entry, ipi_list)
577                 rq->q->mq_ops->complete(rq);
578 }
579 
580 static __latent_entropy void blk_done_softirq(struct softirq_action *h)
581 {
582         blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
583 }
584 
585 static int blk_softirq_cpu_dead(unsigned int cpu)
586 {
587         blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
588         return 0;
589 }
590 
591 static void __blk_mq_complete_request_remote(void *data)
592 {
593         __raise_softirq_irqoff(BLOCK_SOFTIRQ);
594 }
595 
596 static inline bool blk_mq_complete_need_ipi(struct request *rq)
597 {
598         int cpu = raw_smp_processor_id();
599 
600         if (!IS_ENABLED(CONFIG_SMP) ||
601             !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
602                 return false;
603         /*
604          * With force threaded interrupts enabled, raising softirq from an SMP
605          * function call will always result in waking the ksoftirqd thread.
606          * This is probably worse than completing the request on a different
607          * cache domain.
608          */
609         if (force_irqthreads)
610                 return false;
611 
612         /* same CPU or cache domain?  Complete locally */
613         if (cpu == rq->mq_ctx->cpu ||
614             (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
615              cpus_share_cache(cpu, rq->mq_ctx->cpu)))
616                 return false;
617 
618         /* don't try to IPI to an offline CPU */
619         return cpu_online(rq->mq_ctx->cpu);
620 }
621 
622 static void blk_mq_complete_send_ipi(struct request *rq)
623 {
624         struct llist_head *list;
625         unsigned int cpu;
626 
627         cpu = rq->mq_ctx->cpu;
628         list = &per_cpu(blk_cpu_done, cpu);
629         if (llist_add(&rq->ipi_list, list)) {
630                 INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
631                 smp_call_function_single_async(cpu, &rq->csd);
632         }
633 }
634 
635 static void blk_mq_raise_softirq(struct request *rq)
636 {
637         struct llist_head *list;
638 
639         preempt_disable();
640         list = this_cpu_ptr(&blk_cpu_done);
641         if (llist_add(&rq->ipi_list, list))
642                 raise_softirq(BLOCK_SOFTIRQ);
643         preempt_enable();
644 }
645 
646 bool blk_mq_complete_request_remote(struct request *rq)
647 {
648         WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
649 
650         /*
651          * For a polled request, always complete locallly, it's pointless
652          * to redirect the completion.
653          */
654         if (rq->cmd_flags & REQ_HIPRI)
655                 return false;
656 
657         if (blk_mq_complete_need_ipi(rq)) {
658                 blk_mq_complete_send_ipi(rq);
659                 return true;
660         }
661 
662         if (rq->q->nr_hw_queues == 1) {
663                 blk_mq_raise_softirq(rq);
664                 return true;
665         }
666         return false;
667 }
668 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
669 
670 /**
671  * blk_mq_complete_request - end I/O on a request
672  * @rq:         the request being processed
673  *
674  * Description:
675  *      Complete a request by scheduling the ->complete_rq operation.
676  **/
677 void blk_mq_complete_request(struct request *rq)
678 {
679         if (!blk_mq_complete_request_remote(rq))
680                 rq->q->mq_ops->complete(rq);
681 }
682 EXPORT_SYMBOL(blk_mq_complete_request);
683 
684 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
685         __releases(hctx->srcu)
686 {
687         if (!(hctx->flags & BLK_MQ_F_BLOCKING))
688                 rcu_read_unlock();
689         else
690                 srcu_read_unlock(hctx->srcu, srcu_idx);
691 }
692 
693 static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
694         __acquires(hctx->srcu)
695 {
696         if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
697                 /* shut up gcc false positive */
698                 *srcu_idx = 0;
699                 rcu_read_lock();
700         } else
701                 *srcu_idx = srcu_read_lock(hctx->srcu);
702 }
703 
704 /**
705  * blk_mq_start_request - Start processing a request
706  * @rq: Pointer to request to be started
707  *
708  * Function used by device drivers to notify the block layer that a request
709  * is going to be processed now, so blk layer can do proper initializations
710  * such as starting the timeout timer.
711  */
712 void blk_mq_start_request(struct request *rq)
713 {
714         struct request_queue *q = rq->q;
715 
716         trace_block_rq_issue(rq);
717 
718         if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
719                 rq->io_start_time_ns = ktime_get_ns();
720                 rq->stats_sectors = blk_rq_sectors(rq);
721                 rq->rq_flags |= RQF_STATS;
722                 rq_qos_issue(q, rq);
723         }
724 
725         WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
726 
727         blk_add_timer(rq);
728         WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
729 
730 #ifdef CONFIG_BLK_DEV_INTEGRITY
731         if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
732                 q->integrity.profile->prepare_fn(rq);
733 #endif
734 }
735 EXPORT_SYMBOL(blk_mq_start_request);
736 
737 static void __blk_mq_requeue_request(struct request *rq)
738 {
739         struct request_queue *q = rq->q;
740 
741         blk_mq_put_driver_tag(rq);
742 
743         trace_block_rq_requeue(rq);
744         rq_qos_requeue(q, rq);
745 
746         if (blk_mq_request_started(rq)) {
747                 WRITE_ONCE(rq->state, MQ_RQ_IDLE);
748                 rq->rq_flags &= ~RQF_TIMED_OUT;
749         }
750 }
751 
752 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
753 {
754         __blk_mq_requeue_request(rq);
755 
756         /* this request will be re-inserted to io scheduler queue */
757         blk_mq_sched_requeue_request(rq);
758 
759         blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
760 }
761 EXPORT_SYMBOL(blk_mq_requeue_request);
762 
763 static void blk_mq_requeue_work(struct work_struct *work)
764 {
765         struct request_queue *q =
766                 container_of(work, struct request_queue, requeue_work.work);
767         LIST_HEAD(rq_list);
768         struct request *rq, *next;
769 
770         spin_lock_irq(&q->requeue_lock);
771         list_splice_init(&q->requeue_list, &rq_list);
772         spin_unlock_irq(&q->requeue_lock);
773 
774         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
775                 if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP)))
776                         continue;
777 
778                 rq->rq_flags &= ~RQF_SOFTBARRIER;
779                 list_del_init(&rq->queuelist);
780                 /*
781                  * If RQF_DONTPREP, rq has contained some driver specific
782                  * data, so insert it to hctx dispatch list to avoid any
783                  * merge.
784                  */
785                 if (rq->rq_flags & RQF_DONTPREP)
786                         blk_mq_request_bypass_insert(rq, false, false);
787                 else
788                         blk_mq_sched_insert_request(rq, true, false, false);
789         }
790 
791         while (!list_empty(&rq_list)) {
792                 rq = list_entry(rq_list.next, struct request, queuelist);
793                 list_del_init(&rq->queuelist);
794                 blk_mq_sched_insert_request(rq, false, false, false);
795         }
796 
797         blk_mq_run_hw_queues(q, false);
798 }
799 
800 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
801                                 bool kick_requeue_list)
802 {
803         struct request_queue *q = rq->q;
804         unsigned long flags;
805 
806         /*
807          * We abuse this flag that is otherwise used by the I/O scheduler to
808          * request head insertion from the workqueue.
809          */
810         BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
811 
812         spin_lock_irqsave(&q->requeue_lock, flags);
813         if (at_head) {
814                 rq->rq_flags |= RQF_SOFTBARRIER;
815                 list_add(&rq->queuelist, &q->requeue_list);
816         } else {
817                 list_add_tail(&rq->queuelist, &q->requeue_list);
818         }
819         spin_unlock_irqrestore(&q->requeue_lock, flags);
820 
821         if (kick_requeue_list)
822                 blk_mq_kick_requeue_list(q);
823 }
824 
825 void blk_mq_kick_requeue_list(struct request_queue *q)
826 {
827         kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
828 }
829 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
830 
831 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
832                                     unsigned long msecs)
833 {
834         kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
835                                     msecs_to_jiffies(msecs));
836 }
837 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
838 
839 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
840 {
841         if (tag < tags->nr_tags) {
842                 prefetch(tags->rqs[tag]);
843                 return tags->rqs[tag];
844         }
845 
846         return NULL;
847 }
848 EXPORT_SYMBOL(blk_mq_tag_to_rq);
849 
850 static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,
851                                void *priv, bool reserved)
852 {
853         /*
854          * If we find a request that isn't idle and the queue matches,
855          * we know the queue is busy. Return false to stop the iteration.
856          */
857         if (blk_mq_request_started(rq) && rq->q == hctx->queue) {
858                 bool *busy = priv;
859 
860                 *busy = true;
861                 return false;
862         }
863 
864         return true;
865 }
866 
867 bool blk_mq_queue_inflight(struct request_queue *q)
868 {
869         bool busy = false;
870 
871         blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
872         return busy;
873 }
874 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
875 
876 static void blk_mq_rq_timed_out(struct request *req, bool reserved)
877 {
878         req->rq_flags |= RQF_TIMED_OUT;
879         if (req->q->mq_ops->timeout) {
880                 enum blk_eh_timer_return ret;
881 
882                 ret = req->q->mq_ops->timeout(req, reserved);
883                 if (ret == BLK_EH_DONE)
884                         return;
885                 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
886         }
887 
888         blk_add_timer(req);
889 }
890 
891 static bool blk_mq_req_expired(struct request *rq, unsigned long *next)
892 {
893         unsigned long deadline;
894 
895         if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
896                 return false;
897         if (rq->rq_flags & RQF_TIMED_OUT)
898                 return false;
899 
900         deadline = READ_ONCE(rq->deadline);
901         if (time_after_eq(jiffies, deadline))
902                 return true;
903 
904         if (*next == 0)
905                 *next = deadline;
906         else if (time_after(*next, deadline))
907                 *next = deadline;
908         return false;
909 }
910 
911 void blk_mq_put_rq_ref(struct request *rq)
912 {
913         if (is_flush_rq(rq))
914                 rq->end_io(rq, 0);
915         else if (refcount_dec_and_test(&rq->ref))
916                 __blk_mq_free_request(rq);
917 }
918 
919 static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
920                 struct request *rq, void *priv, bool reserved)
921 {
922         unsigned long *next = priv;
923 
924         /*
925          * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
926          * be reallocated underneath the timeout handler's processing, then
927          * the expire check is reliable. If the request is not expired, then
928          * it was completed and reallocated as a new request after returning
929          * from blk_mq_check_expired().
930          */
931         if (blk_mq_req_expired(rq, next))
932                 blk_mq_rq_timed_out(rq, reserved);
933         return true;
934 }
935 
936 static void blk_mq_timeout_work(struct work_struct *work)
937 {
938         struct request_queue *q =
939                 container_of(work, struct request_queue, timeout_work);
940         unsigned long next = 0;
941         struct blk_mq_hw_ctx *hctx;
942         int i;
943 
944         /* A deadlock might occur if a request is stuck requiring a
945          * timeout at the same time a queue freeze is waiting
946          * completion, since the timeout code would not be able to
947          * acquire the queue reference here.
948          *
949          * That's why we don't use blk_queue_enter here; instead, we use
950          * percpu_ref_tryget directly, because we need to be able to
951          * obtain a reference even in the short window between the queue
952          * starting to freeze, by dropping the first reference in
953          * blk_freeze_queue_start, and the moment the last request is
954          * consumed, marked by the instant q_usage_counter reaches
955          * zero.
956          */
957         if (!percpu_ref_tryget(&q->q_usage_counter))
958                 return;
959 
960         blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
961 
962         if (next != 0) {
963                 mod_timer(&q->timeout, next);
964         } else {
965                 /*
966                  * Request timeouts are handled as a forward rolling timer. If
967                  * we end up here it means that no requests are pending and
968                  * also that no request has been pending for a while. Mark
969                  * each hctx as idle.
970                  */
971                 queue_for_each_hw_ctx(q, hctx, i) {
972                         /* the hctx may be unmapped, so check it here */
973                         if (blk_mq_hw_queue_mapped(hctx))
974                                 blk_mq_tag_idle(hctx);
975                 }
976         }
977         blk_queue_exit(q);
978 }
979 
980 struct flush_busy_ctx_data {
981         struct blk_mq_hw_ctx *hctx;
982         struct list_head *list;
983 };
984 
985 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
986 {
987         struct flush_busy_ctx_data *flush_data = data;
988         struct blk_mq_hw_ctx *hctx = flush_data->hctx;
989         struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
990         enum hctx_type type = hctx->type;
991 
992         spin_lock(&ctx->lock);
993         list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
994         sbitmap_clear_bit(sb, bitnr);
995         spin_unlock(&ctx->lock);
996         return true;
997 }
998 
999 /*
1000  * Process software queues that have been marked busy, splicing them
1001  * to the for-dispatch
1002  */
1003 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1004 {
1005         struct flush_busy_ctx_data data = {
1006                 .hctx = hctx,
1007                 .list = list,
1008         };
1009 
1010         sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1011 }
1012 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
1013 
1014 struct dispatch_rq_data {
1015         struct blk_mq_hw_ctx *hctx;
1016         struct request *rq;
1017 };
1018 
1019 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
1020                 void *data)
1021 {
1022         struct dispatch_rq_data *dispatch_data = data;
1023         struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
1024         struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1025         enum hctx_type type = hctx->type;
1026 
1027         spin_lock(&ctx->lock);
1028         if (!list_empty(&ctx->rq_lists[type])) {
1029                 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1030                 list_del_init(&dispatch_data->rq->queuelist);
1031                 if (list_empty(&ctx->rq_lists[type]))
1032                         sbitmap_clear_bit(sb, bitnr);
1033         }
1034         spin_unlock(&ctx->lock);
1035 
1036         return !dispatch_data->rq;
1037 }
1038 
1039 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
1040                                         struct blk_mq_ctx *start)
1041 {
1042         unsigned off = start ? start->index_hw[hctx->type] : 0;
1043         struct dispatch_rq_data data = {
1044                 .hctx = hctx,
1045                 .rq   = NULL,
1046         };
1047 
1048         __sbitmap_for_each_set(&hctx->ctx_map, off,
1049                                dispatch_rq_from_ctx, &data);
1050 
1051         return data.rq;
1052 }
1053 
1054 static inline unsigned int queued_to_index(unsigned int queued)
1055 {
1056         if (!queued)
1057                 return 0;
1058 
1059         return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
1060 }
1061 
1062 static bool __blk_mq_get_driver_tag(struct request *rq)
1063 {
1064         struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags;
1065         unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1066         int tag;
1067 
1068         blk_mq_tag_busy(rq->mq_hctx);
1069 
1070         if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1071                 bt = rq->mq_hctx->tags->breserved_tags;
1072                 tag_offset = 0;
1073         } else {
1074                 if (!hctx_may_queue(rq->mq_hctx, bt))
1075                         return false;
1076         }
1077 
1078         tag = __sbitmap_queue_get(bt);
1079         if (tag == BLK_MQ_NO_TAG)
1080                 return false;
1081 
1082         rq->tag = tag + tag_offset;
1083         return true;
1084 }
1085 
1086 bool blk_mq_get_driver_tag(struct request *rq)
1087 {
1088         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1089 
1090         if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
1091                 return false;
1092 
1093         if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1094                         !(rq->rq_flags & RQF_MQ_INFLIGHT)) {
1095                 rq->rq_flags |= RQF_MQ_INFLIGHT;
1096                 __blk_mq_inc_active_requests(hctx);
1097         }
1098         hctx->tags->rqs[rq->tag] = rq;
1099         return true;
1100 }
1101 
1102 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1103                                 int flags, void *key)
1104 {
1105         struct blk_mq_hw_ctx *hctx;
1106 
1107         hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1108 
1109         spin_lock(&hctx->dispatch_wait_lock);
1110         if (!list_empty(&wait->entry)) {
1111                 struct sbitmap_queue *sbq;
1112 
1113                 list_del_init(&wait->entry);
1114                 sbq = hctx->tags->bitmap_tags;
1115                 atomic_dec(&sbq->ws_active);
1116         }
1117         spin_unlock(&hctx->dispatch_wait_lock);
1118 
1119         blk_mq_run_hw_queue(hctx, true);
1120         return 1;
1121 }
1122 
1123 /*
1124  * Mark us waiting for a tag. For shared tags, this involves hooking us into
1125  * the tag wakeups. For non-shared tags, we can simply mark us needing a
1126  * restart. For both cases, take care to check the condition again after
1127  * marking us as waiting.
1128  */
1129 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1130                                  struct request *rq)
1131 {
1132         struct sbitmap_queue *sbq = hctx->tags->bitmap_tags;
1133         struct wait_queue_head *wq;
1134         wait_queue_entry_t *wait;
1135         bool ret;
1136 
1137         if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
1138                 blk_mq_sched_mark_restart_hctx(hctx);
1139 
1140                 /*
1141                  * It's possible that a tag was freed in the window between the
1142                  * allocation failure and adding the hardware queue to the wait
1143                  * queue.
1144                  *
1145                  * Don't clear RESTART here, someone else could have set it.
1146                  * At most this will cost an extra queue run.
1147                  */
1148                 return blk_mq_get_driver_tag(rq);
1149         }
1150 
1151         wait = &hctx->dispatch_wait;
1152         if (!list_empty_careful(&wait->entry))
1153                 return false;
1154 
1155         wq = &bt_wait_ptr(sbq, hctx)->wait;
1156 
1157         spin_lock_irq(&wq->lock);
1158         spin_lock(&hctx->dispatch_wait_lock);
1159         if (!list_empty(&wait->entry)) {
1160                 spin_unlock(&hctx->dispatch_wait_lock);
1161                 spin_unlock_irq(&wq->lock);
1162                 return false;
1163         }
1164 
1165         atomic_inc(&sbq->ws_active);
1166         wait->flags &= ~WQ_FLAG_EXCLUSIVE;
1167         __add_wait_queue(wq, wait);
1168 
1169         /*
1170          * It's possible that a tag was freed in the window between the
1171          * allocation failure and adding the hardware queue to the wait
1172          * queue.
1173          */
1174         ret = blk_mq_get_driver_tag(rq);
1175         if (!ret) {
1176                 spin_unlock(&hctx->dispatch_wait_lock);
1177                 spin_unlock_irq(&wq->lock);
1178                 return false;
1179         }
1180 
1181         /*
1182          * We got a tag, remove ourselves from the wait queue to ensure
1183          * someone else gets the wakeup.
1184          */
1185         list_del_init(&wait->entry);
1186         atomic_dec(&sbq->ws_active);
1187         spin_unlock(&hctx->dispatch_wait_lock);
1188         spin_unlock_irq(&wq->lock);
1189 
1190         return true;
1191 }
1192 
1193 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT  8
1194 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR  4
1195 /*
1196  * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
1197  * - EWMA is one simple way to compute running average value
1198  * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
1199  * - take 4 as factor for avoiding to get too small(0) result, and this
1200  *   factor doesn't matter because EWMA decreases exponentially
1201  */
1202 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
1203 {
1204         unsigned int ewma;
1205 
1206         ewma = hctx->dispatch_busy;
1207 
1208         if (!ewma && !busy)
1209                 return;
1210 
1211         ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
1212         if (busy)
1213                 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
1214         ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
1215 
1216         hctx->dispatch_busy = ewma;
1217 }
1218 
1219 #define BLK_MQ_RESOURCE_DELAY   3               /* ms units */
1220 
1221 static void blk_mq_handle_dev_resource(struct request *rq,
1222                                        struct list_head *list)
1223 {
1224         struct request *next =
1225                 list_first_entry_or_null(list, struct request, queuelist);
1226 
1227         /*
1228          * If an I/O scheduler has been configured and we got a driver tag for
1229          * the next request already, free it.
1230          */
1231         if (next)
1232                 blk_mq_put_driver_tag(next);
1233 
1234         list_add(&rq->queuelist, list);
1235         __blk_mq_requeue_request(rq);
1236 }
1237 
1238 static void blk_mq_handle_zone_resource(struct request *rq,
1239                                         struct list_head *zone_list)
1240 {
1241         /*
1242          * If we end up here it is because we cannot dispatch a request to a
1243          * specific zone due to LLD level zone-write locking or other zone
1244          * related resource not being available. In this case, set the request
1245          * aside in zone_list for retrying it later.
1246          */
1247         list_add(&rq->queuelist, zone_list);
1248         __blk_mq_requeue_request(rq);
1249 }
1250 
1251 enum prep_dispatch {
1252         PREP_DISPATCH_OK,
1253         PREP_DISPATCH_NO_TAG,
1254         PREP_DISPATCH_NO_BUDGET,
1255 };
1256 
1257 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
1258                                                   bool need_budget)
1259 {
1260         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1261         int budget_token = -1;
1262 
1263         if (need_budget) {
1264                 budget_token = blk_mq_get_dispatch_budget(rq->q);
1265                 if (budget_token < 0) {
1266                         blk_mq_put_driver_tag(rq);
1267                         return PREP_DISPATCH_NO_BUDGET;
1268                 }
1269                 blk_mq_set_rq_budget_token(rq, budget_token);
1270         }
1271 
1272         if (!blk_mq_get_driver_tag(rq)) {
1273                 /*
1274                  * The initial allocation attempt failed, so we need to
1275                  * rerun the hardware queue when a tag is freed. The
1276                  * waitqueue takes care of that. If the queue is run
1277                  * before we add this entry back on the dispatch list,
1278                  * we'll re-run it below.
1279                  */
1280                 if (!blk_mq_mark_tag_wait(hctx, rq)) {
1281                         /*
1282                          * All budgets not got from this function will be put
1283                          * together during handling partial dispatch
1284                          */
1285                         if (need_budget)
1286                                 blk_mq_put_dispatch_budget(rq->q, budget_token);
1287                         return PREP_DISPATCH_NO_TAG;
1288                 }
1289         }
1290 
1291         return PREP_DISPATCH_OK;
1292 }
1293 
1294 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
1295 static void blk_mq_release_budgets(struct request_queue *q,
1296                 struct list_head *list)
1297 {
1298         struct request *rq;
1299 
1300         list_for_each_entry(rq, list, queuelist) {
1301                 int budget_token = blk_mq_get_rq_budget_token(rq);
1302 
1303                 if (budget_token >= 0)
1304                         blk_mq_put_dispatch_budget(q, budget_token);
1305         }
1306 }
1307 
1308 /*
1309  * Returns true if we did some work AND can potentially do more.
1310  */
1311 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
1312                              unsigned int nr_budgets)
1313 {
1314         enum prep_dispatch prep;
1315         struct request_queue *q = hctx->queue;
1316         struct request *rq, *nxt;
1317         int errors, queued;
1318         blk_status_t ret = BLK_STS_OK;
1319         LIST_HEAD(zone_list);
1320         bool needs_resource = false;
1321 
1322         if (list_empty(list))
1323                 return false;
1324 
1325         /*
1326          * Now process all the entries, sending them to the driver.
1327          */
1328         errors = queued = 0;
1329         do {
1330                 struct blk_mq_queue_data bd;
1331 
1332                 rq = list_first_entry(list, struct request, queuelist);
1333 
1334                 WARN_ON_ONCE(hctx != rq->mq_hctx);
1335                 prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
1336                 if (prep != PREP_DISPATCH_OK)
1337                         break;
1338 
1339                 list_del_init(&rq->queuelist);
1340 
1341                 bd.rq = rq;
1342 
1343                 /*
1344                  * Flag last if we have no more requests, or if we have more
1345                  * but can't assign a driver tag to it.
1346                  */
1347                 if (list_empty(list))
1348                         bd.last = true;
1349                 else {
1350                         nxt = list_first_entry(list, struct request, queuelist);
1351                         bd.last = !blk_mq_get_driver_tag(nxt);
1352                 }
1353 
1354                 /*
1355                  * once the request is queued to lld, no need to cover the
1356                  * budget any more
1357                  */
1358                 if (nr_budgets)
1359                         nr_budgets--;
1360                 ret = q->mq_ops->queue_rq(hctx, &bd);
1361                 switch (ret) {
1362                 case BLK_STS_OK:
1363                         queued++;
1364                         break;
1365                 case BLK_STS_RESOURCE:
1366                         needs_resource = true;
1367                         fallthrough;
1368                 case BLK_STS_DEV_RESOURCE:
1369                         blk_mq_handle_dev_resource(rq, list);
1370                         goto out;
1371                 case BLK_STS_ZONE_RESOURCE:
1372                         /*
1373                          * Move the request to zone_list and keep going through
1374                          * the dispatch list to find more requests the drive can
1375                          * accept.
1376                          */
1377                         blk_mq_handle_zone_resource(rq, &zone_list);
1378                         needs_resource = true;
1379                         break;
1380                 default:
1381                         errors++;
1382                         blk_mq_end_request(rq, ret);
1383                 }
1384         } while (!list_empty(list));
1385 out:
1386         if (!list_empty(&zone_list))
1387                 list_splice_tail_init(&zone_list, list);
1388 
1389         hctx->dispatched[queued_to_index(queued)]++;
1390 
1391         /* If we didn't flush the entire list, we could have told the driver
1392          * there was more coming, but that turned out to be a lie.
1393          */
1394         if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued)
1395                 q->mq_ops->commit_rqs(hctx);
1396         /*
1397          * Any items that need requeuing? Stuff them into hctx->dispatch,
1398          * that is where we will continue on next queue run.
1399          */
1400         if (!list_empty(list)) {
1401                 bool needs_restart;
1402                 /* For non-shared tags, the RESTART check will suffice */
1403                 bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
1404                         (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
1405 
1406                 if (nr_budgets)
1407                         blk_mq_release_budgets(q, list);
1408 
1409                 spin_lock(&hctx->lock);
1410                 list_splice_tail_init(list, &hctx->dispatch);
1411                 spin_unlock(&hctx->lock);
1412 
1413                 /*
1414                  * Order adding requests to hctx->dispatch and checking
1415                  * SCHED_RESTART flag. The pair of this smp_mb() is the one
1416                  * in blk_mq_sched_restart(). Avoid restart code path to
1417                  * miss the new added requests to hctx->dispatch, meantime
1418                  * SCHED_RESTART is observed here.
1419                  */
1420                 smp_mb();
1421 
1422                 /*
1423                  * If SCHED_RESTART was set by the caller of this function and
1424                  * it is no longer set that means that it was cleared by another
1425                  * thread and hence that a queue rerun is needed.
1426                  *
1427                  * If 'no_tag' is set, that means that we failed getting
1428                  * a driver tag with an I/O scheduler attached. If our dispatch
1429                  * waitqueue is no longer active, ensure that we run the queue
1430                  * AFTER adding our entries back to the list.
1431                  *
1432                  * If no I/O scheduler has been configured it is possible that
1433                  * the hardware queue got stopped and restarted before requests
1434                  * were pushed back onto the dispatch list. Rerun the queue to
1435                  * avoid starvation. Notes:
1436                  * - blk_mq_run_hw_queue() checks whether or not a queue has
1437                  *   been stopped before rerunning a queue.
1438                  * - Some but not all block drivers stop a queue before
1439                  *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
1440                  *   and dm-rq.
1441                  *
1442                  * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
1443                  * bit is set, run queue after a delay to avoid IO stalls
1444                  * that could otherwise occur if the queue is idle.  We'll do
1445                  * similar if we couldn't get budget or couldn't lock a zone
1446                  * and SCHED_RESTART is set.
1447                  */
1448                 needs_restart = blk_mq_sched_needs_restart(hctx);
1449                 if (prep == PREP_DISPATCH_NO_BUDGET)
1450                         needs_resource = true;
1451                 if (!needs_restart ||
1452                     (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
1453                         blk_mq_run_hw_queue(hctx, true);
1454                 else if (needs_restart && needs_resource)
1455                         blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
1456 
1457                 blk_mq_update_dispatch_busy(hctx, true);
1458                 return false;
1459         } else
1460                 blk_mq_update_dispatch_busy(hctx, false);
1461 
1462         return (queued + errors) != 0;
1463 }
1464 
1465 /**
1466  * __blk_mq_run_hw_queue - Run a hardware queue.
1467  * @hctx: Pointer to the hardware queue to run.
1468  *
1469  * Send pending requests to the hardware.
1470  */
1471 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
1472 {
1473         int srcu_idx;
1474 
1475         /*
1476          * We can't run the queue inline with ints disabled. Ensure that
1477          * we catch bad users of this early.
1478          */
1479         WARN_ON_ONCE(in_interrupt());
1480 
1481         might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1482 
1483         hctx_lock(hctx, &srcu_idx);
1484         blk_mq_sched_dispatch_requests(hctx);
1485         hctx_unlock(hctx, srcu_idx);
1486 }
1487 
1488 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
1489 {
1490         int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
1491 
1492         if (cpu >= nr_cpu_ids)
1493                 cpu = cpumask_first(hctx->cpumask);
1494         return cpu;
1495 }
1496 
1497 /*
1498  * It'd be great if the workqueue API had a way to pass
1499  * in a mask and had some smarts for more clever placement.
1500  * For now we just round-robin here, switching for every
1501  * BLK_MQ_CPU_WORK_BATCH queued items.
1502  */
1503 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
1504 {
1505         bool tried = false;
1506         int next_cpu = hctx->next_cpu;
1507 
1508         if (hctx->queue->nr_hw_queues == 1)
1509                 return WORK_CPU_UNBOUND;
1510 
1511         if (--hctx->next_cpu_batch <= 0) {
1512 select_cpu:
1513                 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
1514                                 cpu_online_mask);
1515                 if (next_cpu >= nr_cpu_ids)
1516                         next_cpu = blk_mq_first_mapped_cpu(hctx);
1517                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1518         }
1519 
1520         /*
1521          * Do unbound schedule if we can't find a online CPU for this hctx,
1522          * and it should only happen in the path of handling CPU DEAD.
1523          */
1524         if (!cpu_online(next_cpu)) {
1525                 if (!tried) {
1526                         tried = true;
1527                         goto select_cpu;
1528                 }
1529 
1530                 /*
1531                  * Make sure to re-select CPU next time once after CPUs
1532                  * in hctx->cpumask become online again.
1533                  */
1534                 hctx->next_cpu = next_cpu;
1535                 hctx->next_cpu_batch = 1;
1536                 return WORK_CPU_UNBOUND;
1537         }
1538 
1539         hctx->next_cpu = next_cpu;
1540         return next_cpu;
1541 }
1542 
1543 /**
1544  * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue.
1545  * @hctx: Pointer to the hardware queue to run.
1546  * @async: If we want to run the queue asynchronously.
1547  * @msecs: Milliseconds of delay to wait before running the queue.
1548  *
1549  * If !@async, try to run the queue now. Else, run the queue asynchronously and
1550  * with a delay of @msecs.
1551  */
1552 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
1553                                         unsigned long msecs)
1554 {
1555         if (unlikely(blk_mq_hctx_stopped(hctx)))
1556                 return;
1557 
1558         if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1559                 int cpu = get_cpu();
1560                 if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1561                         __blk_mq_run_hw_queue(hctx);
1562                         put_cpu();
1563                         return;
1564                 }
1565 
1566                 put_cpu();
1567         }
1568 
1569         kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
1570                                     msecs_to_jiffies(msecs));
1571 }
1572 
1573 /**
1574  * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
1575  * @hctx: Pointer to the hardware queue to run.
1576  * @msecs: Milliseconds of delay to wait before running the queue.
1577  *
1578  * Run a hardware queue asynchronously with a delay of @msecs.
1579  */
1580 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1581 {
1582         __blk_mq_delay_run_hw_queue(hctx, true, msecs);
1583 }
1584 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
1585 
1586 /**
1587  * blk_mq_run_hw_queue - Start to run a hardware queue.
1588  * @hctx: Pointer to the hardware queue to run.
1589  * @async: If we want to run the queue asynchronously.
1590  *
1591  * Check if the request queue is not in a quiesced state and if there are
1592  * pending requests to be sent. If this is true, run the queue to send requests
1593  * to hardware.
1594  */
1595 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1596 {
1597         int srcu_idx;
1598         bool need_run;
1599 
1600         /*
1601          * When queue is quiesced, we may be switching io scheduler, or
1602          * updating nr_hw_queues, or other things, and we can't run queue
1603          * any more, even __blk_mq_hctx_has_pending() can't be called safely.
1604          *
1605          * And queue will be rerun in blk_mq_unquiesce_queue() if it is
1606          * quiesced.
1607          */
1608         hctx_lock(hctx, &srcu_idx);
1609         need_run = !blk_queue_quiesced(hctx->queue) &&
1610                 blk_mq_hctx_has_pending(hctx);
1611         hctx_unlock(hctx, srcu_idx);
1612 
1613         if (need_run)
1614                 __blk_mq_delay_run_hw_queue(hctx, async, 0);
1615 }
1616 EXPORT_SYMBOL(blk_mq_run_hw_queue);
1617 
1618 /*
1619  * Is the request queue handled by an IO scheduler that does not respect
1620  * hardware queues when dispatching?
1621  */
1622 static bool blk_mq_has_sqsched(struct request_queue *q)
1623 {
1624         struct elevator_queue *e = q->elevator;
1625 
1626         if (e && e->type->ops.dispatch_request &&
1627             !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
1628                 return true;
1629         return false;
1630 }
1631 
1632 /*
1633  * Return prefered queue to dispatch from (if any) for non-mq aware IO
1634  * scheduler.
1635  */
1636 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
1637 {
1638         struct blk_mq_hw_ctx *hctx;
1639 
1640         /*
1641          * If the IO scheduler does not respect hardware queues when
1642          * dispatching, we just don't bother with multiple HW queues and
1643          * dispatch from hctx for the current CPU since running multiple queues
1644          * just causes lock contention inside the scheduler and pointless cache
1645          * bouncing.
1646          */
1647         hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT,
1648                                      raw_smp_processor_id());
1649         if (!blk_mq_hctx_stopped(hctx))
1650                 return hctx;
1651         return NULL;
1652 }
1653 
1654 /**
1655  * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
1656  * @q: Pointer to the request queue to run.
1657  * @async: If we want to run the queue asynchronously.
1658  */
1659 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1660 {
1661         struct blk_mq_hw_ctx *hctx, *sq_hctx;
1662         int i;
1663 
1664         sq_hctx = NULL;
1665         if (blk_mq_has_sqsched(q))
1666                 sq_hctx = blk_mq_get_sq_hctx(q);
1667         queue_for_each_hw_ctx(q, hctx, i) {
1668                 if (blk_mq_hctx_stopped(hctx))
1669                         continue;
1670                 /*
1671                  * Dispatch from this hctx either if there's no hctx preferred
1672                  * by IO scheduler or if it has requests that bypass the
1673                  * scheduler.
1674                  */
1675                 if (!sq_hctx || sq_hctx == hctx ||
1676                     !list_empty_careful(&hctx->dispatch))
1677                         blk_mq_run_hw_queue(hctx, async);
1678         }
1679 }
1680 EXPORT_SYMBOL(blk_mq_run_hw_queues);
1681 
1682 /**
1683  * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
1684  * @q: Pointer to the request queue to run.
1685  * @msecs: Milliseconds of delay to wait before running the queues.
1686  */
1687 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
1688 {
1689         struct blk_mq_hw_ctx *hctx, *sq_hctx;
1690         int i;
1691 
1692         sq_hctx = NULL;
1693         if (blk_mq_has_sqsched(q))
1694                 sq_hctx = blk_mq_get_sq_hctx(q);
1695         queue_for_each_hw_ctx(q, hctx, i) {
1696                 if (blk_mq_hctx_stopped(hctx))
1697                         continue;
1698                 /*
1699                  * Dispatch from this hctx either if there's no hctx preferred
1700                  * by IO scheduler or if it has requests that bypass the
1701                  * scheduler.
1702                  */
1703                 if (!sq_hctx || sq_hctx == hctx ||
1704                     !list_empty_careful(&hctx->dispatch))
1705                         blk_mq_delay_run_hw_queue(hctx, msecs);
1706         }
1707 }
1708 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
1709 
1710 /**
1711  * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
1712  * @q: request queue.
1713  *
1714  * The caller is responsible for serializing this function against
1715  * blk_mq_{start,stop}_hw_queue().
1716  */
1717 bool blk_mq_queue_stopped(struct request_queue *q)
1718 {
1719         struct blk_mq_hw_ctx *hctx;
1720         int i;
1721 
1722         queue_for_each_hw_ctx(q, hctx, i)
1723                 if (blk_mq_hctx_stopped(hctx))
1724                         return true;
1725 
1726         return false;
1727 }
1728 EXPORT_SYMBOL(blk_mq_queue_stopped);
1729 
1730 /*
1731  * This function is often used for pausing .queue_rq() by driver when
1732  * there isn't enough resource or some conditions aren't satisfied, and
1733  * BLK_STS_RESOURCE is usually returned.
1734  *
1735  * We do not guarantee that dispatch can be drained or blocked
1736  * after blk_mq_stop_hw_queue() returns. Please use
1737  * blk_mq_quiesce_queue() for that requirement.
1738  */
1739 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
1740 {
1741         cancel_delayed_work(&hctx->run_work);
1742 
1743         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1744 }
1745 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1746 
1747 /*
1748  * This function is often used for pausing .queue_rq() by driver when
1749  * there isn't enough resource or some conditions aren't satisfied, and
1750  * BLK_STS_RESOURCE is usually returned.
1751  *
1752  * We do not guarantee that dispatch can be drained or blocked
1753  * after blk_mq_stop_hw_queues() returns. Please use
1754  * blk_mq_quiesce_queue() for that requirement.
1755  */
1756 void blk_mq_stop_hw_queues(struct request_queue *q)
1757 {
1758         struct blk_mq_hw_ctx *hctx;
1759         int i;
1760 
1761         queue_for_each_hw_ctx(q, hctx, i)
1762                 blk_mq_stop_hw_queue(hctx);
1763 }
1764 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
1765 
1766 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
1767 {
1768         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1769 
1770         blk_mq_run_hw_queue(hctx, false);
1771 }
1772 EXPORT_SYMBOL(blk_mq_start_hw_queue);
1773 
1774 void blk_mq_start_hw_queues(struct request_queue *q)
1775 {
1776         struct blk_mq_hw_ctx *hctx;
1777         int i;
1778 
1779         queue_for_each_hw_ctx(q, hctx, i)
1780                 blk_mq_start_hw_queue(hctx);
1781 }
1782 EXPORT_SYMBOL(blk_mq_start_hw_queues);
1783 
1784 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1785 {
1786         if (!blk_mq_hctx_stopped(hctx))
1787                 return;
1788 
1789         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1790         blk_mq_run_hw_queue(hctx, async);
1791 }
1792 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
1793 
1794 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1795 {
1796         struct blk_mq_hw_ctx *hctx;
1797         int i;
1798 
1799         queue_for_each_hw_ctx(q, hctx, i)
1800                 blk_mq_start_stopped_hw_queue(hctx, async);
1801 }
1802 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
1803 
1804 static void blk_mq_run_work_fn(struct work_struct *work)
1805 {
1806         struct blk_mq_hw_ctx *hctx;
1807 
1808         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
1809 
1810         /*
1811          * If we are stopped, don't run the queue.
1812          */
1813         if (blk_mq_hctx_stopped(hctx))
1814                 return;
1815 
1816         __blk_mq_run_hw_queue(hctx);
1817 }
1818 
1819 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
1820                                             struct request *rq,
1821                                             bool at_head)
1822 {
1823         struct blk_mq_ctx *ctx = rq->mq_ctx;
1824         enum hctx_type type = hctx->type;
1825 
1826         lockdep_assert_held(&ctx->lock);
1827 
1828         trace_block_rq_insert(rq);
1829 
1830         if (at_head)
1831                 list_add(&rq->queuelist, &ctx->rq_lists[type]);
1832         else
1833                 list_add_tail(&rq->queuelist, &ctx->rq_lists[type]);
1834 }
1835 
1836 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
1837                              bool at_head)
1838 {
1839         struct blk_mq_ctx *ctx = rq->mq_ctx;
1840 
1841         lockdep_assert_held(&ctx->lock);
1842 
1843         __blk_mq_insert_req_list(hctx, rq, at_head);
1844         blk_mq_hctx_mark_pending(hctx, ctx);
1845 }
1846 
1847 /**
1848  * blk_mq_request_bypass_insert - Insert a request at dispatch list.
1849  * @rq: Pointer to request to be inserted.
1850  * @at_head: true if the request should be inserted at the head of the list.
1851  * @run_queue: If we should run the hardware queue after inserting the request.
1852  *
1853  * Should only be used carefully, when the caller knows we want to
1854  * bypass a potential IO scheduler on the target device.
1855  */
1856 void blk_mq_request_bypass_insert(struct request *rq, bool at_head,
1857                                   bool run_queue)
1858 {
1859         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1860 
1861         spin_lock(&hctx->lock);
1862         if (at_head)
1863                 list_add(&rq->queuelist, &hctx->dispatch);
1864         else
1865                 list_add_tail(&rq->queuelist, &hctx->dispatch);
1866         spin_unlock(&hctx->lock);
1867 
1868         if (run_queue)
1869                 blk_mq_run_hw_queue(hctx, false);
1870 }
1871 
1872 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
1873                             struct list_head *list)
1874 
1875 {
1876         struct request *rq;
1877         enum hctx_type type = hctx->type;
1878 
1879         /*
1880          * preemption doesn't flush plug list, so it's possible ctx->cpu is
1881          * offline now
1882          */
1883         list_for_each_entry(rq, list, queuelist) {
1884                 BUG_ON(rq->mq_ctx != ctx);
1885                 trace_block_rq_insert(rq);
1886         }
1887 
1888         spin_lock(&ctx->lock);
1889         list_splice_tail_init(list, &ctx->rq_lists[type]);
1890         blk_mq_hctx_mark_pending(hctx, ctx);
1891         spin_unlock(&ctx->lock);
1892 }
1893 
1894 static int plug_rq_cmp(void *priv, const struct list_head *a,
1895                        const struct list_head *b)
1896 {
1897         struct request *rqa = container_of(a, struct request, queuelist);
1898         struct request *rqb = container_of(b, struct request, queuelist);
1899 
1900         if (rqa->mq_ctx != rqb->mq_ctx)
1901                 return rqa->mq_ctx > rqb->mq_ctx;
1902         if (rqa->mq_hctx != rqb->mq_hctx)
1903                 return rqa->mq_hctx > rqb->mq_hctx;
1904 
1905         return blk_rq_pos(rqa) > blk_rq_pos(rqb);
1906 }
1907 
1908 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1909 {
1910         LIST_HEAD(list);
1911 
1912         if (list_empty(&plug->mq_list))
1913                 return;
1914         list_splice_init(&plug->mq_list, &list);
1915 
1916         if (plug->rq_count > 2 && plug->multiple_queues)
1917                 list_sort(NULL, &list, plug_rq_cmp);
1918 
1919         plug->rq_count = 0;
1920 
1921         do {
1922                 struct list_head rq_list;
1923                 struct request *rq, *head_rq = list_entry_rq(list.next);
1924                 struct list_head *pos = &head_rq->queuelist; /* skip first */
1925                 struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx;
1926                 struct blk_mq_ctx *this_ctx = head_rq->mq_ctx;
1927                 unsigned int depth = 1;
1928 
1929                 list_for_each_continue(pos, &list) {
1930                         rq = list_entry_rq(pos);
1931                         BUG_ON(!rq->q);
1932                         if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx)
1933                                 break;
1934                         depth++;
1935                 }
1936 
1937                 list_cut_before(&rq_list, &list, pos);
1938                 trace_block_unplug(head_rq->q, depth, !from_schedule);
1939                 blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list,
1940                                                 from_schedule);
1941         } while(!list_empty(&list));
1942 }
1943 
1944 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
1945                 unsigned int nr_segs)
1946 {
1947         int err;
1948 
1949         if (bio->bi_opf & REQ_RAHEAD)
1950                 rq->cmd_flags |= REQ_FAILFAST_MASK;
1951 
1952         rq->__sector = bio->bi_iter.bi_sector;
1953         rq->write_hint = bio->bi_write_hint;
1954         blk_rq_bio_prep(rq, bio, nr_segs);
1955 
1956         /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
1957         err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
1958         WARN_ON_ONCE(err);
1959 
1960         blk_account_io_start(rq);
1961 }
1962 
1963 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
1964                                             struct request *rq,
1965                                             blk_qc_t *cookie, bool last)
1966 {
1967         struct request_queue *q = rq->q;
1968         struct blk_mq_queue_data bd = {
1969                 .rq = rq,
1970                 .last = last,
1971         };
1972         blk_qc_t new_cookie;
1973         blk_status_t ret;
1974 
1975         new_cookie = request_to_qc_t(hctx, rq);
1976 
1977         /*
1978          * For OK queue, we are done. For error, caller may kill it.
1979          * Any other error (busy), just add it to our list as we
1980          * previously would have done.
1981          */
1982         ret = q->mq_ops->queue_rq(hctx, &bd);
1983         switch (ret) {
1984         case BLK_STS_OK:
1985                 blk_mq_update_dispatch_busy(hctx, false);
1986                 *cookie = new_cookie;
1987                 break;
1988         case BLK_STS_RESOURCE:
1989         case BLK_STS_DEV_RESOURCE:
1990                 blk_mq_update_dispatch_busy(hctx, true);
1991                 __blk_mq_requeue_request(rq);
1992                 break;
1993         default:
1994                 blk_mq_update_dispatch_busy(hctx, false);
1995                 *cookie = BLK_QC_T_NONE;
1996                 break;
1997         }
1998 
1999         return ret;
2000 }
2001 
2002 static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2003                                                 struct request *rq,
2004                                                 blk_qc_t *cookie,
2005                                                 bool bypass_insert, bool last)
2006 {
2007         struct request_queue *q = rq->q;
2008         bool run_queue = true;
2009         int budget_token;
2010 
2011         /*
2012          * RCU or SRCU read lock is needed before checking quiesced flag.
2013          *
2014          * When queue is stopped or quiesced, ignore 'bypass_insert' from
2015          * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller,
2016          * and avoid driver to try to dispatch again.
2017          */
2018         if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) {
2019                 run_queue = false;
2020                 bypass_insert = false;
2021                 goto insert;
2022         }
2023 
2024         if (q->elevator && !bypass_insert)
2025                 goto insert;
2026 
2027         budget_token = blk_mq_get_dispatch_budget(q);
2028         if (budget_token < 0)
2029                 goto insert;
2030 
2031         blk_mq_set_rq_budget_token(rq, budget_token);
2032 
2033         if (!blk_mq_get_driver_tag(rq)) {
2034                 blk_mq_put_dispatch_budget(q, budget_token);
2035                 goto insert;
2036         }
2037 
2038         return __blk_mq_issue_directly(hctx, rq, cookie, last);
2039 insert:
2040         if (bypass_insert)
2041                 return BLK_STS_RESOURCE;
2042 
2043         blk_mq_sched_insert_request(rq, false, run_queue, false);
2044 
2045         return BLK_STS_OK;
2046 }
2047 
2048 /**
2049  * blk_mq_try_issue_directly - Try to send a request directly to device driver.
2050  * @hctx: Pointer of the associated hardware queue.
2051  * @rq: Pointer to request to be sent.
2052  * @cookie: Request queue cookie.
2053  *
2054  * If the device has enough resources to accept a new request now, send the
2055  * request directly to device driver. Else, insert at hctx->dispatch queue, so
2056  * we can try send it another time in the future. Requests inserted at this
2057  * queue have higher priority.
2058  */
2059 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2060                 struct request *rq, blk_qc_t *cookie)
2061 {
2062         blk_status_t ret;
2063         int srcu_idx;
2064 
2065         might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
2066 
2067         hctx_lock(hctx, &srcu_idx);
2068 
2069         ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false, true);
2070         if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE)
2071                 blk_mq_request_bypass_insert(rq, false, true);
2072         else if (ret != BLK_STS_OK)
2073                 blk_mq_end_request(rq, ret);
2074 
2075         hctx_unlock(hctx, srcu_idx);
2076 }
2077 
2078 blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
2079 {
2080         blk_status_t ret;
2081         int srcu_idx;
2082         blk_qc_t unused_cookie;
2083         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2084 
2085         hctx_lock(hctx, &srcu_idx);
2086         ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last);
2087         hctx_unlock(hctx, srcu_idx);
2088 
2089         return ret;
2090 }
2091 
2092 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2093                 struct list_head *list)
2094 {
2095         int queued = 0;
2096         int errors = 0;
2097 
2098         while (!list_empty(list)) {
2099                 blk_status_t ret;
2100                 struct request *rq = list_first_entry(list, struct request,
2101                                 queuelist);
2102 
2103                 list_del_init(&rq->queuelist);
2104                 ret = blk_mq_request_issue_directly(rq, list_empty(list));
2105                 if (ret != BLK_STS_OK) {
2106                         if (ret == BLK_STS_RESOURCE ||
2107                                         ret == BLK_STS_DEV_RESOURCE) {
2108                                 blk_mq_request_bypass_insert(rq, false,
2109                                                         list_empty(list));
2110                                 break;
2111                         }
2112                         blk_mq_end_request(rq, ret);
2113                         errors++;
2114                 } else
2115                         queued++;
2116         }
2117 
2118         /*
2119          * If we didn't flush the entire list, we could have told
2120          * the driver there was more coming, but that turned out to
2121          * be a lie.
2122          */
2123         if ((!list_empty(list) || errors) &&
2124              hctx->queue->mq_ops->commit_rqs && queued)
2125                 hctx->queue->mq_ops->commit_rqs(hctx);
2126 }
2127 
2128 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
2129 {
2130         list_add_tail(&rq->queuelist, &plug->mq_list);
2131         plug->rq_count++;
2132         if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) {
2133                 struct request *tmp;
2134 
2135                 tmp = list_first_entry(&plug->mq_list, struct request,
2136                                                 queuelist);
2137                 if (tmp->q != rq->q)
2138                         plug->multiple_queues = true;
2139         }
2140 }
2141 
2142 /*
2143  * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
2144  * queues. This is important for md arrays to benefit from merging
2145  * requests.
2146  */
2147 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
2148 {
2149         if (plug->multiple_queues)
2150                 return BLK_MAX_REQUEST_COUNT * 2;
2151         return BLK_MAX_REQUEST_COUNT;
2152 }
2153 
2154 /**
2155  * blk_mq_submit_bio - Create and send a request to block device.
2156  * @bio: Bio pointer.
2157  *
2158  * Builds up a request structure from @q and @bio and send to the device. The
2159  * request may not be queued directly to hardware if:
2160  * * This request can be merged with another one
2161  * * We want to place request at plug queue for possible future merging
2162  * * There is an IO scheduler active at this queue
2163  *
2164  * It will not queue the request if there is an error with the bio, or at the
2165  * request creation.
2166  *
2167  * Returns: Request queue cookie.
2168  */
2169 blk_qc_t blk_mq_submit_bio(struct bio *bio)
2170 {
2171         struct request_queue *q = bio->bi_bdev->bd_disk->queue;
2172         const int is_sync = op_is_sync(bio->bi_opf);
2173         const int is_flush_fua = op_is_flush(bio->bi_opf);
2174         struct blk_mq_alloc_data data = {
2175                 .q              = q,
2176         };
2177         struct request *rq;
2178         struct blk_plug *plug;
2179         struct request *same_queue_rq = NULL;
2180         unsigned int nr_segs;
2181         blk_qc_t cookie;
2182         blk_status_t ret;
2183         bool hipri;
2184 
2185         blk_queue_bounce(q, &bio);
2186         __blk_queue_split(&bio, &nr_segs);
2187 
2188         if (!bio_integrity_prep(bio))
2189                 goto queue_exit;
2190 
2191         if (!is_flush_fua && !blk_queue_nomerges(q) &&
2192             blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq))
2193                 goto queue_exit;
2194 
2195         if (blk_mq_sched_bio_merge(q, bio, nr_segs))
2196                 goto queue_exit;
2197 
2198         rq_qos_throttle(q, bio);
2199 
2200         hipri = bio->bi_opf & REQ_HIPRI;
2201 
2202         data.cmd_flags = bio->bi_opf;
2203         rq = __blk_mq_alloc_request(&data);
2204         if (unlikely(!rq)) {
2205                 rq_qos_cleanup(q, bio);
2206                 if (bio->bi_opf & REQ_NOWAIT)
2207                         bio_wouldblock_error(bio);
2208                 goto queue_exit;
2209         }
2210 
2211         trace_block_getrq(bio);
2212 
2213         rq_qos_track(q, rq, bio);
2214 
2215         cookie = request_to_qc_t(data.hctx, rq);
2216 
2217         blk_mq_bio_to_request(rq, bio, nr_segs);
2218 
2219         ret = blk_crypto_init_request(rq);
2220         if (ret != BLK_STS_OK) {
2221                 bio->bi_status = ret;
2222                 bio_endio(bio);
2223                 blk_mq_free_request(rq);
2224                 return BLK_QC_T_NONE;
2225         }
2226 
2227         plug = blk_mq_plug(q, bio);
2228         if (unlikely(is_flush_fua)) {
2229                 /* Bypass scheduler for flush requests */
2230                 blk_insert_flush(rq);
2231                 blk_mq_run_hw_queue(data.hctx, true);
2232         } else if (plug && (q->nr_hw_queues == 1 ||
2233                    blk_mq_is_sbitmap_shared(rq->mq_hctx->flags) ||
2234                    q->mq_ops->commit_rqs || !blk_queue_nonrot(q))) {
2235                 /*
2236                  * Use plugging if we have a ->commit_rqs() hook as well, as
2237                  * we know the driver uses bd->last in a smart fashion.
2238                  *
2239                  * Use normal plugging if this disk is slow HDD, as sequential
2240                  * IO may benefit a lot from plug merging.
2241                  */
2242                 unsigned int request_count = plug->rq_count;
2243                 struct request *last = NULL;
2244 
2245                 if (!request_count)
2246                         trace_block_plug(q);
2247                 else
2248                         last = list_entry_rq(plug->mq_list.prev);
2249 
2250                 if (request_count >= blk_plug_max_rq_count(plug) || (last &&
2251                     blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
2252                         blk_flush_plug_list(plug, false);
2253                         trace_block_plug(q);
2254                 }
2255 
2256                 blk_add_rq_to_plug(plug, rq);
2257         } else if (q->elevator) {
2258                 /* Insert the request at the IO scheduler queue */
2259                 blk_mq_sched_insert_request(rq, false, true, true);
2260         } else if (plug && !blk_queue_nomerges(q)) {
2261                 /*
2262                  * We do limited plugging. If the bio can be merged, do that.
2263                  * Otherwise the existing request in the plug list will be
2264                  * issued. So the plug list will have one request at most
2265                  * The plug list might get flushed before this. If that happens,
2266                  * the plug list is empty, and same_queue_rq is invalid.
2267                  */
2268                 if (list_empty(&plug->mq_list))
2269                         same_queue_rq = NULL;
2270                 if (same_queue_rq) {
2271                         list_del_init(&same_queue_rq->queuelist);
2272                         plug->rq_count--;
2273                 }
2274                 blk_add_rq_to_plug(plug, rq);
2275                 trace_block_plug(q);
2276 
2277                 if (same_queue_rq) {
2278                         data.hctx = same_queue_rq->mq_hctx;
2279                         trace_block_unplug(q, 1, true);
2280                         blk_mq_try_issue_directly(data.hctx, same_queue_rq,
2281                                         &cookie);
2282                 }
2283         } else if ((q->nr_hw_queues > 1 && is_sync) ||
2284                         !data.hctx->dispatch_busy) {
2285                 /*
2286                  * There is no scheduler and we can try to send directly
2287                  * to the hardware.
2288                  */
2289                 blk_mq_try_issue_directly(data.hctx, rq, &cookie);
2290         } else {
2291                 /* Default case. */
2292                 blk_mq_sched_insert_request(rq, false, true, true);
2293         }
2294 
2295         if (!hipri)
2296                 return BLK_QC_T_NONE;
2297         return cookie;
2298 queue_exit:
2299         blk_queue_exit(q);
2300         return BLK_QC_T_NONE;
2301 }
2302 
2303 static size_t order_to_size(unsigned int order)
2304 {
2305         return (size_t)PAGE_SIZE << order;
2306 }
2307 
2308 /* called before freeing request pool in @tags */
2309 static void blk_mq_clear_rq_mapping(struct blk_mq_tag_set *set,
2310                 struct blk_mq_tags *tags, unsigned int hctx_idx)
2311 {
2312         struct blk_mq_tags *drv_tags = set->tags[hctx_idx];
2313         struct page *page;
2314         unsigned long flags;
2315 
2316         list_for_each_entry(page, &tags->page_list, lru) {
2317                 unsigned long start = (unsigned long)page_address(page);
2318                 unsigned long end = start + order_to_size(page->private);
2319                 int i;
2320 
2321                 for (i = 0; i < set->queue_depth; i++) {
2322                         struct request *rq = drv_tags->rqs[i];
2323                         unsigned long rq_addr = (unsigned long)rq;
2324 
2325                         if (rq_addr >= start && rq_addr < end) {
2326                                 WARN_ON_ONCE(refcount_read(&rq->ref) != 0);
2327                                 cmpxchg(&drv_tags->rqs[i], rq, NULL);
2328                         }
2329                 }
2330         }
2331 
2332         /*
2333          * Wait until all pending iteration is done.
2334          *
2335          * Request reference is cleared and it is guaranteed to be observed
2336          * after the ->lock is released.
2337          */
2338         spin_lock_irqsave(&drv_tags->lock, flags);
2339         spin_unlock_irqrestore(&drv_tags->lock, flags);
2340 }
2341 
2342 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
2343                      unsigned int hctx_idx)
2344 {
2345         struct page *page;
2346 
2347         if (tags->rqs && set->ops->exit_request) {
2348                 int i;
2349 
2350                 for (i = 0; i < tags->nr_tags; i++) {
2351                         struct request *rq = tags->static_rqs[i];
2352 
2353                         if (!rq)
2354                                 continue;
2355                         set->ops->exit_request(set, rq, hctx_idx);
2356                         tags->static_rqs[i] = NULL;
2357                 }
2358         }
2359 
2360         blk_mq_clear_rq_mapping(set, tags, hctx_idx);
2361 
2362         while (!list_empty(&tags->page_list)) {
2363                 page = list_first_entry(&tags->page_list, struct page, lru);
2364                 list_del_init(&page->lru);
2365                 /*
2366                  * Remove kmemleak object previously allocated in
2367                  * blk_mq_alloc_rqs().
2368                  */
2369                 kmemleak_free(page_address(page));
2370                 __free_pages(page, page->private);
2371         }
2372 }
2373 
2374 void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags)
2375 {
2376         kfree(tags->rqs);
2377         tags->rqs = NULL;
2378         kfree(tags->static_rqs);
2379         tags->static_rqs = NULL;
2380 
2381         blk_mq_free_tags(tags, flags);
2382 }
2383 
2384 struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
2385                                         unsigned int hctx_idx,
2386                                         unsigned int nr_tags,
2387                                         unsigned int reserved_tags,
2388                                         unsigned int flags)
2389 {
2390         struct blk_mq_tags *tags;
2391         int node;
2392 
2393         node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2394         if (node == NUMA_NO_NODE)
2395                 node = set->numa_node;
2396 
2397         tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
2398         if (!tags)
2399                 return NULL;
2400 
2401         tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2402                                  GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2403                                  node);
2404         if (!tags->rqs) {
2405                 blk_mq_free_tags(tags, flags);
2406                 return NULL;
2407         }
2408 
2409         tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2410                                         GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2411                                         node);
2412         if (!tags->static_rqs) {
2413                 kfree(tags->rqs);
2414                 blk_mq_free_tags(tags, flags);
2415                 return NULL;
2416         }
2417 
2418         return tags;
2419 }
2420 
2421 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
2422                                unsigned int hctx_idx, int node)
2423 {
2424         int ret;
2425 
2426         if (set->ops->init_request) {
2427                 ret = set->ops->init_request(set, rq, hctx_idx, node);
2428                 if (ret)
2429                         return ret;
2430         }
2431 
2432         WRITE_ONCE(rq->state, MQ_RQ_IDLE);
2433         return 0;
2434 }
2435 
2436 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
2437                      unsigned int hctx_idx, unsigned int depth)
2438 {
2439         unsigned int i, j, entries_per_page, max_order = 4;
2440         size_t rq_size, left;
2441         int node;
2442 
2443         node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2444         if (node == NUMA_NO_NODE)
2445                 node = set->numa_node;
2446 
2447         INIT_LIST_HEAD(&tags->page_list);
2448 
2449         /*
2450          * rq_size is the size of the request plus driver payload, rounded
2451          * to the cacheline size
2452          */
2453         rq_size = round_up(sizeof(struct request) + set->cmd_size,
2454                                 cache_line_size());
2455         left = rq_size * depth;
2456 
2457         for (i = 0; i < depth; ) {
2458                 int this_order = max_order;
2459                 struct page *page;
2460                 int to_do;
2461                 void *p;
2462 
2463                 while (this_order && left < order_to_size(this_order - 1))
2464                         this_order--;
2465 
2466                 do {
2467                         page = alloc_pages_node(node,
2468                                 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
2469                                 this_order);
2470                         if (page)
2471                                 break;
2472                         if (!this_order--)
2473                                 break;
2474                         if (order_to_size(this_order) < rq_size)
2475                                 break;
2476                 } while (1);
2477 
2478                 if (!page)
2479                         goto fail;
2480 
2481                 page->private = this_order;
2482                 list_add_tail(&page->lru, &tags->page_list);
2483 
2484                 p = page_address(page);
2485                 /*
2486                  * Allow kmemleak to scan these pages as they contain pointers
2487                  * to additional allocations like via ops->init_request().
2488                  */
2489                 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
2490                 entries_per_page = order_to_size(this_order) / rq_size;
2491                 to_do = min(entries_per_page, depth - i);
2492                 left -= to_do * rq_size;
2493                 for (j = 0; j < to_do; j++) {
2494                         struct request *rq = p;
2495 
2496                         tags->static_rqs[i] = rq;
2497                         if (blk_mq_init_request(set, rq, hctx_idx, node)) {
2498                                 tags->static_rqs[i] = NULL;
2499                                 goto fail;
2500                         }
2501 
2502                         p += rq_size;
2503                         i++;
2504                 }
2505         }
2506         return 0;
2507 
2508 fail:
2509         blk_mq_free_rqs(set, tags, hctx_idx);
2510         return -ENOMEM;
2511 }
2512 
2513 struct rq_iter_data {
2514         struct blk_mq_hw_ctx *hctx;
2515         bool has_rq;
2516 };
2517 
2518 static bool blk_mq_has_request(struct request *rq, void *data, bool reserved)
2519 {
2520         struct rq_iter_data *iter_data = data;
2521 
2522         if (rq->mq_hctx != iter_data->hctx)
2523                 return true;
2524         iter_data->has_rq = true;
2525         return false;
2526 }
2527 
2528 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
2529 {
2530         struct blk_mq_tags *tags = hctx->sched_tags ?
2531                         hctx->sched_tags : hctx->tags;
2532         struct rq_iter_data data = {
2533                 .hctx   = hctx,
2534         };
2535 
2536         blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
2537         return data.has_rq;
2538 }
2539 
2540 static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
2541                 struct blk_mq_hw_ctx *hctx)
2542 {
2543         if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu)
2544                 return false;
2545         if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
2546                 return false;
2547         return true;
2548 }
2549 
2550 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
2551 {
2552         struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2553                         struct blk_mq_hw_ctx, cpuhp_online);
2554 
2555         if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
2556             !blk_mq_last_cpu_in_hctx(cpu, hctx))
2557                 return 0;
2558 
2559         /*
2560          * Prevent new request from being allocated on the current hctx.
2561          *
2562          * The smp_mb__after_atomic() Pairs with the implied barrier in
2563          * test_and_set_bit_lock in sbitmap_get().  Ensures the inactive flag is
2564          * seen once we return from the tag allocator.
2565          */
2566         set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2567         smp_mb__after_atomic();
2568 
2569         /*
2570          * Try to grab a reference to the queue and wait for any outstanding
2571          * requests.  If we could not grab a reference the queue has been
2572          * frozen and there are no requests.
2573          */
2574         if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
2575                 while (blk_mq_hctx_has_requests(hctx))
2576                         msleep(5);
2577                 percpu_ref_put(&hctx->queue->q_usage_counter);
2578         }
2579 
2580         return 0;
2581 }
2582 
2583 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
2584 {
2585         struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
2586                         struct blk_mq_hw_ctx, cpuhp_online);
2587 
2588         if (cpumask_test_cpu(cpu, hctx->cpumask))
2589                 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
2590         return 0;
2591 }
2592 
2593 /*
2594  * 'cpu' is going away. splice any existing rq_list entries from this
2595  * software queue to the hw queue dispatch list, and ensure that it
2596  * gets run.
2597  */
2598 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
2599 {
2600         struct blk_mq_hw_ctx *hctx;
2601         struct blk_mq_ctx *ctx;
2602         LIST_HEAD(tmp);
2603         enum hctx_type type;
2604 
2605         hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
2606         if (!cpumask_test_cpu(cpu, hctx->cpumask))
2607                 return 0;
2608 
2609         ctx = __blk_mq_get_ctx(hctx->queue, cpu);
2610         type = hctx->type;
2611 
2612         spin_lock(&ctx->lock);
2613         if (!list_empty(&ctx->rq_lists[type])) {
2614                 list_splice_init(&ctx->rq_lists[type], &tmp);
2615                 blk_mq_hctx_clear_pending(hctx, ctx);
2616         }
2617         spin_unlock(&ctx->lock);
2618 
2619         if (list_empty(&tmp))
2620                 return 0;
2621 
2622         spin_lock(&hctx->lock);
2623         list_splice_tail_init(&tmp, &hctx->dispatch);
2624         spin_unlock(&hctx->lock);
2625 
2626         blk_mq_run_hw_queue(hctx, true);
2627         return 0;
2628 }
2629 
2630 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
2631 {
2632         if (!(hctx->flags & BLK_MQ_F_STACKING))
2633                 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2634                                                     &hctx->cpuhp_online);
2635         cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
2636                                             &hctx->cpuhp_dead);
2637 }
2638 
2639 /*
2640  * Before freeing hw queue, clearing the flush request reference in
2641  * tags->rqs[] for avoiding potential UAF.
2642  */
2643 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
2644                 unsigned int queue_depth, struct request *flush_rq)
2645 {
2646         int i;
2647         unsigned long flags;
2648 
2649         /* The hw queue may not be mapped yet */
2650         if (!tags)
2651                 return;
2652 
2653         WARN_ON_ONCE(refcount_read(&flush_rq->ref) != 0);
2654 
2655         for (i = 0; i < queue_depth; i++)
2656                 cmpxchg(&tags->rqs[i], flush_rq, NULL);
2657 
2658         /*
2659          * Wait until all pending iteration is done.
2660          *
2661          * Request reference is cleared and it is guaranteed to be observed
2662          * after the ->lock is released.
2663          */
2664         spin_lock_irqsave(&tags->lock, flags);
2665         spin_unlock_irqrestore(&tags->lock, flags);
2666 }
2667 
2668 /* hctx->ctxs will be freed in queue's release handler */
2669 static void blk_mq_exit_hctx(struct request_queue *q,
2670                 struct blk_mq_tag_set *set,
2671                 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
2672 {
2673         struct request *flush_rq = hctx->fq->flush_rq;
2674 
2675         if (blk_mq_hw_queue_mapped(hctx))
2676                 blk_mq_tag_idle(hctx);
2677 
2678         blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
2679                         set->queue_depth, flush_rq);
2680         if (set->ops->exit_request)
2681                 set->ops->exit_request(set, flush_rq, hctx_idx);
2682 
2683         if (set->ops->exit_hctx)
2684                 set->ops->exit_hctx(hctx, hctx_idx);
2685 
2686         blk_mq_remove_cpuhp(hctx);
2687 
2688         spin_lock(&q->unused_hctx_lock);
2689         list_add(&hctx->hctx_list, &q->unused_hctx_list);
2690         spin_unlock(&q->unused_hctx_lock);
2691 }
2692 
2693 static void blk_mq_exit_hw_queues(struct request_queue *q,
2694                 struct blk_mq_tag_set *set, int nr_queue)
2695 {
2696         struct blk_mq_hw_ctx *hctx;
2697         unsigned int i;
2698 
2699         queue_for_each_hw_ctx(q, hctx, i) {
2700                 if (i == nr_queue)
2701                         break;
2702                 blk_mq_debugfs_unregister_hctx(hctx);
2703                 blk_mq_exit_hctx(q, set, hctx, i);
2704         }
2705 }
2706 
2707 static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
2708 {
2709         int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);
2710 
2711         BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu),
2712                            __alignof__(struct blk_mq_hw_ctx)) !=
2713                      sizeof(struct blk_mq_hw_ctx));
2714 
2715         if (tag_set->flags & BLK_MQ_F_BLOCKING)
2716                 hw_ctx_size += sizeof(struct srcu_struct);
2717 
2718         return hw_ctx_size;
2719 }
2720 
2721 static int blk_mq_init_hctx(struct request_queue *q,
2722                 struct blk_mq_tag_set *set,
2723                 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
2724 {
2725         hctx->queue_num = hctx_idx;
2726 
2727         if (!(hctx->flags & BLK_MQ_F_STACKING))
2728                 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
2729                                 &hctx->cpuhp_online);
2730         cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
2731 
2732         hctx->tags = set->tags[hctx_idx];
2733 
2734         if (set->ops->init_hctx &&
2735             set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
2736                 goto unregister_cpu_notifier;
2737 
2738         if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
2739                                 hctx->numa_node))
2740                 goto exit_hctx;
2741         return 0;
2742 
2743  exit_hctx:
2744         if (set->ops->exit_hctx)
2745                 set->ops->exit_hctx(hctx, hctx_idx);
2746  unregister_cpu_notifier:
2747         blk_mq_remove_cpuhp(hctx);
2748         return -1;
2749 }
2750 
2751 static struct blk_mq_hw_ctx *
2752 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
2753                 int node)
2754 {
2755         struct blk_mq_hw_ctx *hctx;
2756         gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
2757 
2758         hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node);
2759         if (!hctx)
2760                 goto fail_alloc_hctx;
2761 
2762         if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
2763                 goto free_hctx;
2764 
2765         atomic_set(&hctx->nr_active, 0);
2766         if (node == NUMA_NO_NODE)
2767                 node = set->numa_node;
2768         hctx->numa_node = node;
2769 
2770         INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
2771         spin_lock_init(&hctx->lock);
2772         INIT_LIST_HEAD(&hctx->dispatch);
2773         hctx->queue = q;
2774         hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
2775 
2776         INIT_LIST_HEAD(&hctx->hctx_list);
2777 
2778         /*
2779          * Allocate space for all possible cpus to avoid allocation at
2780          * runtime
2781          */
2782         hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
2783                         gfp, node);
2784         if (!hctx->ctxs)
2785                 goto free_cpumask;
2786 
2787         if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
2788                                 gfp, node, false, false))
2789                 goto free_ctxs;
2790         hctx->nr_ctx = 0;
2791 
2792         spin_lock_init(&hctx->dispatch_wait_lock);
2793         init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
2794         INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
2795 
2796         hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
2797         if (!hctx->fq)
2798                 goto free_bitmap;
2799 
2800         if (hctx->flags & BLK_MQ_F_BLOCKING)
2801                 init_srcu_struct(hctx->srcu);
2802         blk_mq_hctx_kobj_init(hctx);
2803 
2804         return hctx;
2805 
2806  free_bitmap:
2807         sbitmap_free(&hctx->ctx_map);
2808  free_ctxs:
2809         kfree(hctx->ctxs);
2810  free_cpumask:
2811         free_cpumask_var(hctx->cpumask);
2812  free_hctx:
2813         kfree(hctx);
2814  fail_alloc_hctx:
2815         return NULL;
2816 }
2817 
2818 static void blk_mq_init_cpu_queues(struct request_queue *q,
2819                                    unsigned int nr_hw_queues)
2820 {
2821         struct blk_mq_tag_set *set = q->tag_set;
2822         unsigned int i, j;
2823 
2824         for_each_possible_cpu(i) {
2825                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
2826                 struct blk_mq_hw_ctx *hctx;
2827                 int k;
2828 
2829                 __ctx->cpu = i;
2830                 spin_lock_init(&__ctx->lock);
2831                 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
2832                         INIT_LIST_HEAD(&__ctx->rq_lists[k]);
2833 
2834                 __ctx->queue = q;
2835 
2836                 /*
2837                  * Set local node, IFF we have more than one hw queue. If
2838                  * not, we remain on the home node of the device
2839                  */
2840                 for (j = 0; j < set->nr_maps; j++) {
2841                         hctx = blk_mq_map_queue_type(q, j, i);
2842                         if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
2843                                 hctx->numa_node = cpu_to_node(i);
2844                 }
2845         }
2846 }
2847 
2848 static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set,
2849                                         int hctx_idx)
2850 {
2851         unsigned int flags = set->flags;
2852         int ret = 0;
2853 
2854         set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2855                                         set->queue_depth, set->reserved_tags, flags);
2856         if (!set->tags[hctx_idx])
2857                 return false;
2858 
2859         ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
2860                                 set->queue_depth);
2861         if (!ret)
2862                 return true;
2863 
2864         blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2865         set->tags[hctx_idx] = NULL;
2866         return false;
2867 }
2868 
2869 static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2870                                          unsigned int hctx_idx)
2871 {
2872         unsigned int flags = set->flags;
2873 
2874         if (set->tags && set->tags[hctx_idx]) {
2875                 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2876                 blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2877                 set->tags[hctx_idx] = NULL;
2878         }
2879 }
2880 
2881 static void blk_mq_map_swqueue(struct request_queue *q)
2882 {
2883         unsigned int i, j, hctx_idx;
2884         struct blk_mq_hw_ctx *hctx;
2885         struct blk_mq_ctx *ctx;
2886         struct blk_mq_tag_set *set = q->tag_set;
2887 
2888         queue_for_each_hw_ctx(q, hctx, i) {
2889                 cpumask_clear(hctx->cpumask);
2890                 hctx->nr_ctx = 0;
2891                 hctx->dispatch_from = NULL;
2892         }
2893 
2894         /*
2895          * Map software to hardware queues.
2896          *
2897          * If the cpu isn't present, the cpu is mapped to first hctx.
2898          */
2899         for_each_possible_cpu(i) {
2900 
2901                 ctx = per_cpu_ptr(q->queue_ctx, i);
2902                 for (j = 0; j < set->nr_maps; j++) {
2903                         if (!set->map[j].nr_queues) {
2904                                 ctx->hctxs[j] = blk_mq_map_queue_type(q,
2905                                                 HCTX_TYPE_DEFAULT, i);
2906                                 continue;
2907                         }
2908                         hctx_idx = set->map[j].mq_map[i];
2909                         /* unmapped hw queue can be remapped after CPU topo changed */
2910                         if (!set->tags[hctx_idx] &&
2911                             !__blk_mq_alloc_map_and_request(set, hctx_idx)) {
2912                                 /*
2913                                  * If tags initialization fail for some hctx,
2914                                  * that hctx won't be brought online.  In this
2915                                  * case, remap the current ctx to hctx[0] which
2916                                  * is guaranteed to always have tags allocated
2917                                  */
2918                                 set->map[j].mq_map[i] = 0;
2919                         }
2920 
2921                         hctx = blk_mq_map_queue_type(q, j, i);
2922                         ctx->hctxs[j] = hctx;
2923                         /*
2924                          * If the CPU is already set in the mask, then we've
2925                          * mapped this one already. This can happen if
2926                          * devices share queues across queue maps.
2927                          */
2928                         if (cpumask_test_cpu(i, hctx->cpumask))
2929                                 continue;
2930 
2931                         cpumask_set_cpu(i, hctx->cpumask);
2932                         hctx->type = j;
2933                         ctx->index_hw[hctx->type] = hctx->nr_ctx;
2934                         hctx->ctxs[hctx->nr_ctx++] = ctx;
2935 
2936                         /*
2937                          * If the nr_ctx type overflows, we have exceeded the
2938                          * amount of sw queues we can support.
2939                          */
2940                         BUG_ON(!hctx->nr_ctx);
2941                 }
2942 
2943                 for (; j < HCTX_MAX_TYPES; j++)
2944                         ctx->hctxs[j] = blk_mq_map_queue_type(q,
2945                                         HCTX_TYPE_DEFAULT, i);
2946         }
2947 
2948         queue_for_each_hw_ctx(q, hctx, i) {
2949                 /*
2950                  * If no software queues are mapped to this hardware queue,
2951                  * disable it and free the request entries.
2952                  */
2953                 if (!hctx->nr_ctx) {
2954                         /* Never unmap queue 0.  We need it as a
2955                          * fallback in case of a new remap fails
2956                          * allocation
2957                          */
2958                         if (i && set->tags[i])
2959                                 blk_mq_free_map_and_requests(set, i);
2960 
2961                         hctx->tags = NULL;
2962                         continue;
2963                 }
2964 
2965                 hctx->tags = set->tags[i];
2966                 WARN_ON(!hctx->tags);
2967 
2968                 /*
2969                  * Set the map size to the number of mapped software queues.
2970                  * This is more accurate and more efficient than looping
2971                  * over all possibly mapped software queues.
2972                  */
2973                 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2974 
2975                 /*
2976                  * Initialize batch roundrobin counts
2977                  */
2978                 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
2979                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2980         }
2981 }
2982 
2983 /*
2984  * Caller needs to ensure that we're either frozen/quiesced, or that
2985  * the queue isn't live yet.
2986  */
2987 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2988 {
2989         struct blk_mq_hw_ctx *hctx;
2990         int i;
2991 
2992         queue_for_each_hw_ctx(q, hctx, i) {
2993                 if (shared) {
2994                         hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
2995                 } else {
2996                         blk_mq_tag_idle(hctx);
2997                         hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
2998                 }
2999         }
3000 }
3001 
3002 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
3003                                          bool shared)
3004 {
3005         struct request_queue *q;
3006 
3007         lockdep_assert_held(&set->tag_list_lock);
3008 
3009         list_for_each_entry(q, &set->tag_list, tag_set_list) {
3010                 blk_mq_freeze_queue(q);
3011                 queue_set_hctx_shared(q, shared);
3012                 blk_mq_unfreeze_queue(q);
3013         }
3014 }
3015 
3016 static void blk_mq_del_queue_tag_set(struct request_queue *q)
3017 {
3018         struct blk_mq_tag_set *set = q->tag_set;
3019 
3020         mutex_lock(&set->tag_list_lock);
3021         list_del(&q->tag_set_list);
3022         if (list_is_singular(&set->tag_list)) {
3023                 /* just transitioned to unshared */
3024                 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
3025                 /* update existing queue */
3026                 blk_mq_update_tag_set_shared(set, false);
3027         }
3028         mutex_unlock(&set->tag_list_lock);
3029         INIT_LIST_HEAD(&q->tag_set_list);
3030 }
3031 
3032 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
3033                                      struct request_queue *q)
3034 {
3035         mutex_lock(&set->tag_list_lock);
3036 
3037         /*
3038          * Check to see if we're transitioning to shared (from 1 to 2 queues).
3039          */
3040         if (!list_empty(&set->tag_list) &&
3041             !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
3042                 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
3043                 /* update existing queue */
3044                 blk_mq_update_tag_set_shared(set, true);
3045         }
3046         if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
3047                 queue_set_hctx_shared(q, true);
3048         list_add_tail(&q->tag_set_list, &set->tag_list);
3049 
3050         mutex_unlock(&set->tag_list_lock);
3051 }
3052 
3053 /* All allocations will be freed in release handler of q->mq_kobj */
3054 static int blk_mq_alloc_ctxs(struct request_queue *q)
3055 {
3056         struct blk_mq_ctxs *ctxs;
3057         int cpu;
3058 
3059         ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
3060         if (!ctxs)
3061                 return -ENOMEM;
3062 
3063         ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
3064         if (!ctxs->queue_ctx)
3065                 goto fail;
3066 
3067         for_each_possible_cpu(cpu) {
3068                 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
3069                 ctx->ctxs = ctxs;
3070         }
3071 
3072         q->mq_kobj = &ctxs->kobj;
3073         q->queue_ctx = ctxs->queue_ctx;
3074 
3075         return 0;
3076  fail:
3077         kfree(ctxs);
3078         return -ENOMEM;
3079 }
3080 
3081 /*
3082  * It is the actual release handler for mq, but we do it from
3083  * request queue's release handler for avoiding use-after-free
3084  * and headache because q->mq_kobj shouldn't have been introduced,
3085  * but we can't group ctx/kctx kobj without it.
3086  */
3087 void blk_mq_release(struct request_queue *q)
3088 {
3089         struct blk_mq_hw_ctx *hctx, *next;
3090         int i;
3091 
3092         queue_for_each_hw_ctx(q, hctx, i)
3093                 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
3094 
3095         /* all hctx are in .unused_hctx_list now */
3096         list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
3097                 list_del_init(&hctx->hctx_list);
3098                 kobject_put(&hctx->kobj);
3099         }
3100 
3101         kfree(q->queue_hw_ctx);
3102 
3103         /*
3104          * release .mq_kobj and sw queue's kobject now because
3105          * both share lifetime with request queue.
3106          */
3107         blk_mq_sysfs_deinit(q);
3108 }
3109 
3110 static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
3111                 void *queuedata)
3112 {
3113         struct request_queue *q;
3114         int ret;
3115 
3116         q = blk_alloc_queue(set->numa_node);
3117         if (!q)
3118                 return ERR_PTR(-ENOMEM);
3119         q->queuedata = queuedata;
3120         ret = blk_mq_init_allocated_queue(set, q);
3121         if (ret) {
3122                 blk_cleanup_queue(q);
3123                 return ERR_PTR(ret);
3124         }
3125         return q;
3126 }
3127 
3128 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
3129 {
3130         return blk_mq_init_queue_data(set, NULL);
3131 }
3132 EXPORT_SYMBOL(blk_mq_init_queue);
3133 
3134 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata)
3135 {
3136         struct request_queue *q;
3137         struct gendisk *disk;
3138 
3139         q = blk_mq_init_queue_data(set, queuedata);
3140         if (IS_ERR(q))
3141                 return ERR_CAST(q);
3142 
3143         disk = __alloc_disk_node(0, set->numa_node);
3144         if (!disk) {
3145                 blk_cleanup_queue(q);
3146                 return ERR_PTR(-ENOMEM);
3147         }
3148         disk->queue = q;
3149         return disk;
3150 }
3151 EXPORT_SYMBOL(__blk_mq_alloc_disk);
3152 
3153 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
3154                 struct blk_mq_tag_set *set, struct request_queue *q,
3155                 int hctx_idx, int node)
3156 {
3157         struct blk_mq_hw_ctx *hctx = NULL, *tmp;
3158 
3159         /* reuse dead hctx first */
3160         spin_lock(&q->unused_hctx_lock);
3161         list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
3162                 if (tmp->numa_node == node) {
3163                         hctx = tmp;
3164                         break;
3165                 }
3166         }
3167         if (hctx)
3168                 list_del_init(&hctx->hctx_list);
3169         spin_unlock(&q->unused_hctx_lock);
3170 
3171         if (!hctx)
3172                 hctx = blk_mq_alloc_hctx(q, set, node);
3173         if (!hctx)
3174                 goto fail;
3175 
3176         if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
3177                 goto free_hctx;
3178 
3179         return hctx;
3180 
3181  free_hctx:
3182         kobject_put(&hctx->kobj);
3183  fail:
3184         return NULL;
3185 }
3186 
3187 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
3188                                                 struct request_queue *q)
3189 {
3190         int i, j, end;
3191         struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
3192 
3193         if (q->nr_hw_queues < set->nr_hw_queues) {
3194                 struct blk_mq_hw_ctx **new_hctxs;
3195 
3196                 new_hctxs = kcalloc_node(set->nr_hw_queues,
3197                                        sizeof(*new_hctxs), GFP_KERNEL,
3198                                        set->numa_node);
3199                 if (!new_hctxs)
3200                         return;
3201                 if (hctxs)
3202                         memcpy(new_hctxs, hctxs, q->nr_hw_queues *
3203                                sizeof(*hctxs));
3204                 q->queue_hw_ctx = new_hctxs;
3205                 kfree(hctxs);
3206                 hctxs = new_hctxs;
3207         }
3208 
3209         /* protect against switching io scheduler  */
3210         mutex_lock(&q->sysfs_lock);
3211         for (i = 0; i < set->nr_hw_queues; i++) {
3212                 int node;
3213                 struct blk_mq_hw_ctx *hctx;
3214 
3215                 node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], i);
3216                 /*
3217                  * If the hw queue has been mapped to another numa node,
3218                  * we need to realloc the hctx. If allocation fails, fallback
3219                  * to use the previous one.
3220                  */
3221                 if (hctxs[i] && (hctxs[i]->numa_node == node))
3222                         continue;
3223 
3224                 hctx = blk_mq_alloc_and_init_hctx(set, q, i, node);
3225                 if (hctx) {
3226                         if (hctxs[i])
3227                                 blk_mq_exit_hctx(q, set, hctxs[i], i);
3228                         hctxs[i] = hctx;
3229                 } else {
3230                         if (hctxs[i])
3231                                 pr_warn("Allocate new hctx on node %d fails,\
3232                                                 fallback to previous one on node %d\n",
3233                                                 node, hctxs[i]->numa_node);
3234                         else
3235                                 break;
3236                 }
3237         }
3238         /*
3239          * Increasing nr_hw_queues fails. Free the newly allocated
3240          * hctxs and keep the previous q->nr_hw_queues.
3241          */
3242         if (i != set->nr_hw_queues) {
3243                 j = q->nr_hw_queues;
3244                 end = i;
3245         } else {
3246                 j = i;
3247                 end = q->nr_hw_queues;
3248                 q->nr_hw_queues = set->nr_hw_queues;
3249         }
3250 
3251         for (; j < end; j++) {
3252                 struct blk_mq_hw_ctx *hctx = hctxs[j];
3253 
3254                 if (hctx) {
3255                         if (hctx->tags)
3256                                 blk_mq_free_map_and_requests(set, j);
3257                         blk_mq_exit_hctx(q, set, hctx, j);
3258                         hctxs[j] = NULL;
3259                 }
3260         }
3261         mutex_unlock(&q->sysfs_lock);
3262 }
3263 
3264 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
3265                 struct request_queue *q)
3266 {
3267         /* mark the queue as mq asap */
3268         q->mq_ops = set->ops;
3269 
3270         q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
3271                                              blk_mq_poll_stats_bkt,
3272                                              BLK_MQ_POLL_STATS_BKTS, q);
3273         if (!q->poll_cb)
3274                 goto err_exit;
3275 
3276         if (blk_mq_alloc_ctxs(q))
3277                 goto err_poll;
3278 
3279         /* init q->mq_kobj and sw queues' kobjects */
3280         blk_mq_sysfs_init(q);
3281 
3282         INIT_LIST_HEAD(&q->unused_hctx_list);
3283         spin_lock_init(&q->unused_hctx_lock);
3284 
3285         blk_mq_realloc_hw_ctxs(set, q);
3286         if (!q->nr_hw_queues)
3287                 goto err_hctxs;
3288 
3289         INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
3290         blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
3291 
3292         q->tag_set = set;
3293 
3294         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
3295         if (set->nr_maps > HCTX_TYPE_POLL &&
3296             set->map[HCTX_TYPE_POLL].nr_queues)
3297                 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
3298 
3299         q->sg_reserved_size = INT_MAX;
3300 
3301         INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
3302         INIT_LIST_HEAD(&q->requeue_list);
3303         spin_lock_init(&q->requeue_lock);
3304 
3305         q->nr_requests = set->queue_depth;
3306 
3307         /*
3308          * Default to classic polling
3309          */
3310         q->poll_nsec = BLK_MQ_POLL_CLASSIC;
3311 
3312         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
3313         blk_mq_add_queue_tag_set(set, q);
3314         blk_mq_map_swqueue(q);
3315         return 0;
3316 
3317 err_hctxs:
3318         kfree(q->queue_hw_ctx);
3319         q->nr_hw_queues = 0;
3320         blk_mq_sysfs_deinit(q);
3321 err_poll:
3322         blk_stat_free_callback(q->poll_cb);
3323         q->poll_cb = NULL;
3324 err_exit:
3325         q->mq_ops = NULL;
3326         return -ENOMEM;
3327 }
3328 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
3329 
3330 /* tags can _not_ be used after returning from blk_mq_exit_queue */
3331 void blk_mq_exit_queue(struct request_queue *q)
3332 {
3333         struct blk_mq_tag_set *set = q->tag_set;
3334 
3335         /* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
3336         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
3337         /* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
3338         blk_mq_del_queue_tag_set(q);
3339 }
3340 
3341 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
3342 {
3343         int i;
3344 
3345         for (i = 0; i < set->nr_hw_queues; i++) {
3346                 if (!__blk_mq_alloc_map_and_request(set, i))
3347                         goto out_unwind;
3348                 cond_resched();
3349         }
3350 
3351         return 0;
3352 
3353 out_unwind:
3354         while (--i >= 0)
3355                 blk_mq_free_map_and_requests(set, i);
3356 
3357         return -ENOMEM;
3358 }
3359 
3360 /*
3361  * Allocate the request maps associated with this tag_set. Note that this
3362  * may reduce the depth asked for, if memory is tight. set->queue_depth
3363  * will be updated to reflect the allocated depth.
3364  */
3365 static int blk_mq_alloc_map_and_requests(struct blk_mq_tag_set *set)
3366 {
3367         unsigned int depth;
3368         int err;
3369 
3370         depth = set->queue_depth;
3371         do {
3372                 err = __blk_mq_alloc_rq_maps(set);
3373                 if (!err)
3374                         break;
3375 
3376                 set->queue_depth >>= 1;
3377                 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
3378                         err = -ENOMEM;
3379                         break;
3380                 }
3381         } while (set->queue_depth);
3382 
3383         if (!set->queue_depth || err) {
3384                 pr_err("blk-mq: failed to allocate request map\n");
3385                 return -ENOMEM;
3386         }
3387 
3388         if (depth != set->queue_depth)
3389                 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
3390                                                 depth, set->queue_depth);
3391 
3392         return 0;
3393 }
3394 
3395 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
3396 {
3397         /*
3398          * blk_mq_map_queues() and multiple .map_queues() implementations
3399          * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
3400          * number of hardware queues.
3401          */
3402         if (set->nr_maps == 1)
3403                 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
3404 
3405         if (set->ops->map_queues && !is_kdump_kernel()) {
3406                 int i;
3407 
3408                 /*
3409                  * transport .map_queues is usually done in the following
3410                  * way:
3411                  *
3412                  * for (queue = 0; queue < set->nr_hw_queues; queue++) {
3413                  *      mask = get_cpu_mask(queue)
3414                  *      for_each_cpu(cpu, mask)
3415                  *              set->map[x].mq_map[cpu] = queue;
3416                  * }
3417                  *
3418                  * When we need to remap, the table has to be cleared for
3419                  * killing stale mapping since one CPU may not be mapped
3420                  * to any hw queue.
3421                  */
3422                 for (i = 0; i < set->nr_maps; i++)
3423                         blk_mq_clear_mq_map(&set->map[i]);
3424 
3425                 return set->ops->map_queues(set);
3426         } else {
3427                 BUG_ON(set->nr_maps > 1);
3428                 return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
3429         }
3430 }
3431 
3432 static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
3433                                   int cur_nr_hw_queues, int new_nr_hw_queues)
3434 {
3435         struct blk_mq_tags **new_tags;
3436 
3437         if (cur_nr_hw_queues >= new_nr_hw_queues)
3438                 return 0;
3439 
3440         new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
3441                                 GFP_KERNEL, set->numa_node);
3442         if (!new_tags)
3443                 return -ENOMEM;
3444 
3445         if (set->tags)
3446                 memcpy(new_tags, set->tags, cur_nr_hw_queues *
3447                        sizeof(*set->tags));
3448         kfree(set->tags);
3449         set->tags = new_tags;
3450         set->nr_hw_queues = new_nr_hw_queues;
3451 
3452         return 0;
3453 }
3454 
3455 static int blk_mq_alloc_tag_set_tags(struct blk_mq_tag_set *set,
3456                                 int new_nr_hw_queues)
3457 {
3458         return blk_mq_realloc_tag_set_tags(set, 0, new_nr_hw_queues);
3459 }
3460 
3461 /*
3462  * Alloc a tag set to be associated with one or more request queues.
3463  * May fail with EINVAL for various error conditions. May adjust the
3464  * requested depth down, if it's too large. In that case, the set
3465  * value will be stored in set->queue_depth.
3466  */
3467 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
3468 {
3469         int i, ret;
3470 
3471         BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
3472 
3473         if (!set->nr_hw_queues)
3474                 return -EINVAL;
3475         if (!set->queue_depth)
3476                 return -EINVAL;
3477         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
3478                 return -EINVAL;
3479 
3480         if (!set->ops->queue_rq)
3481                 return -EINVAL;
3482 
3483         if (!set->ops->get_budget ^ !set->ops->put_budget)
3484                 return -EINVAL;
3485 
3486         if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
3487                 pr_info("blk-mq: reduced tag depth to %u\n",
3488                         BLK_MQ_MAX_DEPTH);
3489                 set->queue_depth = BLK_MQ_MAX_DEPTH;
3490         }
3491 
3492         if (!set->nr_maps)
3493                 set->nr_maps = 1;
3494         else if (set->nr_maps > HCTX_MAX_TYPES)
3495                 return -EINVAL;
3496 
3497         /*
3498          * If a crashdump is active, then we are potentially in a very
3499          * memory constrained environment. Limit us to 1 queue and
3500          * 64 tags to prevent using too much memory.
3501          */
3502         if (is_kdump_kernel()) {
3503                 set->nr_hw_queues = 1;
3504                 set->nr_maps = 1;
3505                 set->queue_depth = min(64U, set->queue_depth);
3506         }
3507         /*
3508          * There is no use for more h/w queues than cpus if we just have
3509          * a single map
3510          */
3511         if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
3512                 set->nr_hw_queues = nr_cpu_ids;
3513 
3514         if (blk_mq_alloc_tag_set_tags(set, set->nr_hw_queues) < 0)
3515                 return -ENOMEM;
3516 
3517         ret = -ENOMEM;
3518         for (i = 0; i < set->nr_maps; i++) {
3519                 set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
3520                                                   sizeof(set->map[i].mq_map[0]),
3521                                                   GFP_KERNEL, set->numa_node);
3522                 if (!set->map[i].mq_map)
3523                         goto out_free_mq_map;
3524                 set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues;
3525         }
3526 
3527         ret = blk_mq_update_queue_map(set);
3528         if (ret)
3529                 goto out_free_mq_map;
3530 
3531         ret = blk_mq_alloc_map_and_requests(set);
3532         if (ret)
3533                 goto out_free_mq_map;
3534 
3535         if (blk_mq_is_sbitmap_shared(set->flags)) {
3536                 atomic_set(&set->active_queues_shared_sbitmap, 0);
3537 
3538                 if (blk_mq_init_shared_sbitmap(set)) {
3539                         ret = -ENOMEM;
3540                         goto out_free_mq_rq_maps;
3541                 }
3542         }
3543 
3544         mutex_init(&set->tag_list_lock);
3545         INIT_LIST_HEAD(&set->tag_list);
3546 
3547         return 0;
3548 
3549 out_free_mq_rq_maps:
3550         for (i = 0; i < set->nr_hw_queues; i++)
3551                 blk_mq_free_map_and_requests(set, i);
3552 out_free_mq_map:
3553         for (i = 0; i < set->nr_maps; i++) {
3554                 kfree(set->map[i].mq_map);
3555                 set->map[i].mq_map = NULL;
3556         }
3557         kfree(set->tags);
3558         set->tags = NULL;
3559         return ret;
3560 }
3561 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
3562 
3563 /* allocate and initialize a tagset for a simple single-queue device */
3564 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
3565                 const struct blk_mq_ops *ops, unsigned int queue_depth,
3566                 unsigned int set_flags)
3567 {
3568         memset(set, 0, sizeof(*set));
3569         set->ops = ops;
3570         set->nr_hw_queues = 1;
3571         set->nr_maps = 1;
3572         set->queue_depth = queue_depth;
3573         set->numa_node = NUMA_NO_NODE;
3574         set->flags = set_flags;
3575         return blk_mq_alloc_tag_set(set);
3576 }
3577 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
3578 
3579 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
3580 {
3581         int i, j;
3582 
3583         for (i = 0; i < set->nr_hw_queues; i++)
3584                 blk_mq_free_map_and_requests(set, i);
3585 
3586         if (blk_mq_is_sbitmap_shared(set->flags))
3587                 blk_mq_exit_shared_sbitmap(set);
3588 
3589         for (j = 0; j < set->nr_maps; j++) {
3590                 kfree(set->map[j].mq_map);
3591                 set->map[j].mq_map = NULL;
3592         }
3593 
3594         kfree(set->tags);
3595         set->tags = NULL;
3596 }
3597 EXPORT_SYMBOL(blk_mq_free_tag_set);
3598 
3599 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
3600 {
3601         struct blk_mq_tag_set *set = q->tag_set;
3602         struct blk_mq_hw_ctx *hctx;
3603         int i, ret;
3604 
3605         if (!set)
3606                 return -EINVAL;
3607 
3608         if (q->nr_requests == nr)
3609                 return 0;
3610 
3611         blk_mq_freeze_queue(q);
3612         blk_mq_quiesce_queue(q);
3613 
3614         ret = 0;
3615         queue_for_each_hw_ctx(q, hctx, i) {
3616                 if (!hctx->tags)
3617                         continue;
3618                 /*
3619                  * If we're using an MQ scheduler, just update the scheduler
3620                  * queue depth. This is similar to what the old code would do.
3621                  */
3622                 if (!hctx->sched_tags) {
3623                         ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
3624                                                         false);
3625                         if (!ret && blk_mq_is_sbitmap_shared(set->flags))
3626                                 blk_mq_tag_resize_shared_sbitmap(set, nr);
3627                 } else {
3628                         ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
3629                                                         nr, true);
3630                         if (blk_mq_is_sbitmap_shared(set->flags)) {
3631                                 hctx->sched_tags->bitmap_tags =
3632                                         &q->sched_bitmap_tags;
3633                                 hctx->sched_tags->breserved_tags =
3634                                         &q->sched_breserved_tags;
3635                         }
3636                 }
3637                 if (ret)
3638                         break;
3639                 if (q->elevator && q->elevator->type->ops.depth_updated)
3640                         q->elevator->type->ops.depth_updated(hctx);
3641         }
3642         if (!ret) {
3643                 q->nr_requests = nr;
3644                 if (q->elevator && blk_mq_is_sbitmap_shared(set->flags))
3645                         sbitmap_queue_resize(&q->sched_bitmap_tags,
3646                                              nr - set->reserved_tags);
3647         }
3648 
3649         blk_mq_unquiesce_queue(q);
3650         blk_mq_unfreeze_queue(q);
3651 
3652         return ret;
3653 }
3654 
3655 /*
3656  * request_queue and elevator_type pair.
3657  * It is just used by __blk_mq_update_nr_hw_queues to cache
3658  * the elevator_type associated with a request_queue.
3659  */
3660 struct blk_mq_qe_pair {
3661         struct list_head node;
3662         struct request_queue *q;
3663         struct elevator_type *type;
3664 };
3665 
3666 /*
3667  * Cache the elevator_type in qe pair list and switch the
3668  * io scheduler to 'none'
3669  */
3670 static bool blk_mq_elv_switch_none(struct list_head *head,
3671                 struct request_queue *q)
3672 {
3673         struct blk_mq_qe_pair *qe;
3674 
3675         if (!q->elevator)
3676                 return true;
3677 
3678         qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
3679         if (!qe)
3680                 return false;
3681 
3682         INIT_LIST_HEAD(&qe->node);
3683         qe->q = q;
3684         qe->type = q->elevator->type;
3685         list_add(&qe->node, head);
3686 
3687         mutex_lock(&q->sysfs_lock);
3688         /*
3689          * After elevator_switch_mq, the previous elevator_queue will be
3690          * released by elevator_release. The reference of the io scheduler
3691          * module get by elevator_get will also be put. So we need to get
3692          * a reference of the io scheduler module here to prevent it to be
3693          * removed.
3694          */
3695         __module_get(qe->type->elevator_owner);
3696         elevator_switch_mq(q, NULL);
3697         mutex_unlock(&q->sysfs_lock);
3698 
3699         return true;
3700 }
3701 
3702 static void blk_mq_elv_switch_back(struct list_head *head,
3703                 struct request_queue *q)
3704 {
3705         struct blk_mq_qe_pair *qe;
3706         struct elevator_type *t = NULL;
3707 
3708         list_for_each_entry(qe, head, node)
3709                 if (qe->q == q) {
3710                         t = qe->type;
3711                         break;
3712                 }
3713 
3714         if (!t)
3715                 return;
3716 
3717         list_del(&qe->node);
3718         kfree(qe);
3719 
3720         mutex_lock(&q->sysfs_lock);
3721         elevator_switch_mq(q, t);
3722         mutex_unlock(&q->sysfs_lock);
3723 }
3724 
3725 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
3726                                                         int nr_hw_queues)
3727 {
3728         struct request_queue *q;
3729         LIST_HEAD(head);
3730         int prev_nr_hw_queues;
3731 
3732         lockdep_assert_held(&set->tag_list_lock);
3733 
3734         if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
3735                 nr_hw_queues = nr_cpu_ids;
3736         if (nr_hw_queues < 1)
3737                 return;
3738         if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
3739                 return;
3740 
3741         list_for_each_entry(q, &set->tag_list, tag_set_list)
3742                 blk_mq_freeze_queue(q);
3743         /*
3744          * Switch IO scheduler to 'none', cleaning up the data associated
3745          * with the previous scheduler. We will switch back once we are done
3746          * updating the new sw to hw queue mappings.
3747          */
3748         list_for_each_entry(q, &set->tag_list, tag_set_list)
3749                 if (!blk_mq_elv_switch_none(&head, q))
3750                         goto switch_back;
3751 
3752         list_for_each_entry(q, &set->tag_list, tag_set_list) {
3753                 blk_mq_debugfs_unregister_hctxs(q);
3754                 blk_mq_sysfs_unregister(q);
3755         }
3756 
3757         prev_nr_hw_queues = set->nr_hw_queues;
3758         if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) <
3759             0)
3760                 goto reregister;
3761 
3762         set->nr_hw_queues = nr_hw_queues;
3763 fallback:
3764         blk_mq_update_queue_map(set);
3765         list_for_each_entry(q, &set->tag_list, tag_set_list) {
3766                 blk_mq_realloc_hw_ctxs(set, q);
3767                 if (q->nr_hw_queues != set->nr_hw_queues) {
3768                         pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
3769                                         nr_hw_queues, prev_nr_hw_queues);
3770                         set->nr_hw_queues = prev_nr_hw_queues;
3771                         blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
3772                         goto fallback;
3773                 }
3774                 blk_mq_map_swqueue(q);
3775         }
3776 
3777 reregister:
3778         list_for_each_entry(q, &set->tag_list, tag_set_list) {
3779                 blk_mq_sysfs_register(q);
3780                 blk_mq_debugfs_register_hctxs(q);
3781         }
3782 
3783 switch_back:
3784         list_for_each_entry(q, &set->tag_list, tag_set_list)
3785                 blk_mq_elv_switch_back(&head, q);
3786 
3787         list_for_each_entry(q, &set->tag_list, tag_set_list)
3788                 blk_mq_unfreeze_queue(q);
3789 }
3790 
3791 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
3792 {
3793         mutex_lock(&set->tag_list_lock);
3794         __blk_mq_update_nr_hw_queues(set, nr_hw_queues);
3795         mutex_unlock(&set->tag_list_lock);
3796 }
3797 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
3798 
3799 /* Enable polling stats and return whether they were already enabled. */
3800 static bool blk_poll_stats_enable(struct request_queue *q)
3801 {
3802         if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
3803             blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q))
3804                 return true;
3805         blk_stat_add_callback(q, q->poll_cb);
3806         return false;
3807 }
3808 
3809 static void blk_mq_poll_stats_start(struct request_queue *q)
3810 {
3811         /*
3812          * We don't arm the callback if polling stats are not enabled or the
3813          * callback is already active.
3814          */
3815         if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
3816             blk_stat_is_active(q->poll_cb))
3817                 return;
3818 
3819         blk_stat_activate_msecs(q->poll_cb, 100);
3820 }
3821 
3822 static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
3823 {
3824         struct request_queue *q = cb->data;
3825         int bucket;
3826 
3827         for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
3828                 if (cb->stat[bucket].nr_samples)
3829                         q->poll_stat[bucket] = cb->stat[bucket];
3830         }
3831 }
3832 
3833 static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
3834                                        struct request *rq)
3835 {
3836         unsigned long ret = 0;
3837         int bucket;
3838 
3839         /*
3840          * If stats collection isn't on, don't sleep but turn it on for
3841          * future users
3842          */
3843         if (!blk_poll_stats_enable(q))
3844                 return 0;
3845 
3846         /*
3847          * As an optimistic guess, use half of the mean service time
3848          * for this type of request. We can (and should) make this smarter.
3849          * For instance, if the completion latencies are tight, we can
3850          * get closer than just half the mean. This is especially
3851          * important on devices where the completion latencies are longer
3852          * than ~10 usec. We do use the stats for the relevant IO size
3853          * if available which does lead to better estimates.
3854          */
3855         bucket = blk_mq_poll_stats_bkt(rq);
3856         if (bucket < 0)
3857                 return ret;
3858 
3859         if (q->poll_stat[bucket].nr_samples)
3860                 ret = (q->poll_stat[bucket].mean + 1) / 2;
3861 
3862         return ret;
3863 }
3864 
3865 static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
3866                                      struct request *rq)
3867 {
3868         struct hrtimer_sleeper hs;
3869         enum hrtimer_mode mode;
3870         unsigned int nsecs;
3871         ktime_t kt;
3872 
3873         if (rq->rq_flags & RQF_MQ_POLL_SLEPT)
3874                 return false;
3875 
3876         /*
3877          * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
3878          *
3879          *  0:  use half of prev avg
3880          * >0:  use this specific value
3881          */
3882         if (q->poll_nsec > 0)
3883                 nsecs = q->poll_nsec;
3884         else
3885                 nsecs = blk_mq_poll_nsecs(q, rq);
3886 
3887         if (!nsecs)
3888                 return false;
3889 
3890         rq->rq_flags |= RQF_MQ_POLL_SLEPT;
3891 
3892         /*
3893          * This will be replaced with the stats tracking code, using
3894          * 'avg_completion_time / 2' as the pre-sleep target.
3895          */
3896         kt = nsecs;
3897 
3898         mode = HRTIMER_MODE_REL;
3899         hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
3900         hrtimer_set_expires(&hs.timer, kt);
3901 
3902         do {
3903                 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
3904                         break;
3905                 set_current_state(TASK_UNINTERRUPTIBLE);
3906                 hrtimer_sleeper_start_expires(&hs, mode);
3907                 if (hs.task)
3908                         io_schedule();
3909                 hrtimer_cancel(&hs.timer);
3910                 mode = HRTIMER_MODE_ABS;
3911         } while (hs.task && !signal_pending(current));
3912 
3913         __set_current_state(TASK_RUNNING);
3914         destroy_hrtimer_on_stack(&hs.timer);
3915         return true;
3916 }
3917 
3918 static bool blk_mq_poll_hybrid(struct request_queue *q,
3919                                struct blk_mq_hw_ctx *hctx, blk_qc_t cookie)
3920 {
3921         struct request *rq;
3922 
3923         if (q->poll_nsec == BLK_MQ_POLL_CLASSIC)
3924                 return false;
3925 
3926         if (!blk_qc_t_is_internal(cookie))
3927                 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
3928         else {
3929                 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
3930                 /*
3931                  * With scheduling, if the request has completed, we'll
3932                  * get a NULL return here, as we clear the sched tag when
3933                  * that happens. The request still remains valid, like always,
3934                  * so we should be safe with just the NULL check.
3935                  */
3936                 if (!rq)
3937                         return false;
3938         }
3939 
3940         return blk_mq_poll_hybrid_sleep(q, rq);
3941 }
3942 
3943 /**
3944  * blk_poll - poll for IO completions
3945  * @q:  the queue
3946  * @cookie: cookie passed back at IO submission time
3947  * @spin: whether to spin for completions
3948  *
3949  * Description:
3950  *    Poll for completions on the passed in queue. Returns number of
3951  *    completed entries found. If @spin is true, then blk_poll will continue
3952  *    looping until at least one completion is found, unless the task is
3953  *    otherwise marked running (or we need to reschedule).
3954  */
3955 int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
3956 {
3957         struct blk_mq_hw_ctx *hctx;
3958         unsigned int state;
3959 
3960         if (!blk_qc_t_valid(cookie) ||
3961             !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
3962                 return 0;
3963 
3964         if (current->plug)
3965                 blk_flush_plug_list(current->plug, false);
3966 
3967         hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
3968 
3969         /*
3970          * If we sleep, have the caller restart the poll loop to reset
3971          * the state. Like for the other success return cases, the
3972          * caller is responsible for checking if the IO completed. If
3973          * the IO isn't complete, we'll get called again and will go
3974          * straight to the busy poll loop. If specified not to spin,
3975          * we also should not sleep.
3976          */
3977         if (spin && blk_mq_poll_hybrid(q, hctx, cookie))
3978                 return 1;
3979 
3980         hctx->poll_considered++;
3981 
3982         state = get_current_state();
3983         do {
3984                 int ret;
3985 
3986                 hctx->poll_invoked++;
3987 
3988                 ret = q->mq_ops->poll(hctx);
3989                 if (ret > 0) {
3990                         hctx->poll_success++;
3991                         __set_current_state(TASK_RUNNING);
3992                         return ret;
3993                 }
3994 
3995                 if (signal_pending_state(state, current))
3996                         __set_current_state(TASK_RUNNING);
3997 
3998                 if (task_is_running(current))
3999                         return 1;
4000                 if (ret < 0 || !spin)
4001                         break;
4002                 cpu_relax();
4003         } while (!need_resched());
4004 
4005         __set_current_state(TASK_RUNNING);
4006         return 0;
4007 }
4008 EXPORT_SYMBOL_GPL(blk_poll);
4009 
4010 unsigned int blk_mq_rq_cpu(struct request *rq)
4011 {
4012         return rq->mq_ctx->cpu;
4013 }
4014 EXPORT_SYMBOL(blk_mq_rq_cpu);
4015 
4016 static int __init blk_mq_init(void)
4017 {
4018         int i;
4019 
4020         for_each_possible_cpu(i)
4021                 init_llist_head(&per_cpu(blk_cpu_done, i));
4022         open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
4023 
4024         cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
4025                                   "block/softirq:dead", NULL,
4026                                   blk_softirq_cpu_dead);
4027         cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
4028                                 blk_mq_hctx_notify_dead);
4029         cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
4030                                 blk_mq_hctx_notify_online,
4031                                 blk_mq_hctx_notify_offline);
4032         return 0;
4033 }
4034 subsys_initcall(blk_mq_init);
4035 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp