~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/block/blk-mq-tag.c

Version: ~ [ linux-5.15-rc1 ] ~ [ linux-5.14.5 ] ~ [ linux-5.13.18 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.66 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.147 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.206 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.246 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.282 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.283 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

Differences between /block/blk-mq-tag.c (Version linux-4.17.19) and /block/blk-mq-tag.c (Version linux-4.10.17)


  1 /*                                                  1 /*
  2  * Tag allocation using scalable bitmaps. Uses      2  * Tag allocation using scalable bitmaps. Uses active queue tracking to support
  3  * fairer distribution of tags between multipl      3  * fairer distribution of tags between multiple submitters when a shared tag map
  4  * is used.                                         4  * is used.
  5  *                                                  5  *
  6  * Copyright (C) 2013-2014 Jens Axboe               6  * Copyright (C) 2013-2014 Jens Axboe
  7  */                                                 7  */
  8 #include <linux/kernel.h>                           8 #include <linux/kernel.h>
  9 #include <linux/module.h>                           9 #include <linux/module.h>
 10                                                    10 
 11 #include <linux/blk-mq.h>                          11 #include <linux/blk-mq.h>
 12 #include "blk.h"                                   12 #include "blk.h"
 13 #include "blk-mq.h"                                13 #include "blk-mq.h"
 14 #include "blk-mq-tag.h"                            14 #include "blk-mq-tag.h"
 15                                                    15 
 16 bool blk_mq_has_free_tags(struct blk_mq_tags *     16 bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
 17 {                                                  17 {
 18         if (!tags)                                 18         if (!tags)
 19                 return true;                       19                 return true;
 20                                                    20 
 21         return sbitmap_any_bit_clear(&tags->bi     21         return sbitmap_any_bit_clear(&tags->bitmap_tags.sb);
 22 }                                                  22 }
 23                                                    23 
 24 /*                                                 24 /*
 25  * If a previously inactive queue goes active,     25  * If a previously inactive queue goes active, bump the active user count.
 26  */                                                26  */
 27 bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *h     27 bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
 28 {                                                  28 {
 29         if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hc     29         if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
 30             !test_and_set_bit(BLK_MQ_S_TAG_ACT     30             !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
 31                 atomic_inc(&hctx->tags->active     31                 atomic_inc(&hctx->tags->active_queues);
 32                                                    32 
 33         return true;                               33         return true;
 34 }                                                  34 }
 35                                                    35 
 36 /*                                                 36 /*
 37  * Wakeup all potentially sleeping on tags         37  * Wakeup all potentially sleeping on tags
 38  */                                                38  */
 39 void blk_mq_tag_wakeup_all(struct blk_mq_tags      39 void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
 40 {                                                  40 {
 41         sbitmap_queue_wake_all(&tags->bitmap_t     41         sbitmap_queue_wake_all(&tags->bitmap_tags);
 42         if (include_reserve)                       42         if (include_reserve)
 43                 sbitmap_queue_wake_all(&tags->     43                 sbitmap_queue_wake_all(&tags->breserved_tags);
 44 }                                                  44 }
 45                                                    45 
 46 /*                                                 46 /*
 47  * If a previously busy queue goes inactive, p     47  * If a previously busy queue goes inactive, potential waiters could now
 48  * be allowed to queue. Wake them up and check     48  * be allowed to queue. Wake them up and check.
 49  */                                                49  */
 50 void __blk_mq_tag_idle(struct blk_mq_hw_ctx *h     50 void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
 51 {                                                  51 {
 52         struct blk_mq_tags *tags = hctx->tags;     52         struct blk_mq_tags *tags = hctx->tags;
 53                                                    53 
 54         if (!test_and_clear_bit(BLK_MQ_S_TAG_A     54         if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
 55                 return;                            55                 return;
 56                                                    56 
 57         atomic_dec(&tags->active_queues);          57         atomic_dec(&tags->active_queues);
 58                                                    58 
 59         blk_mq_tag_wakeup_all(tags, false);        59         blk_mq_tag_wakeup_all(tags, false);
 60 }                                                  60 }
 61                                                    61 
 62 /*                                                 62 /*
 63  * For shared tag users, we track the number o     63  * For shared tag users, we track the number of currently active users
 64  * and attempt to provide a fair share of the      64  * and attempt to provide a fair share of the tag depth for each of them.
 65  */                                                65  */
 66 static inline bool hctx_may_queue(struct blk_m     66 static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
 67                                   struct sbitm     67                                   struct sbitmap_queue *bt)
 68 {                                                  68 {
 69         unsigned int depth, users;                 69         unsigned int depth, users;
 70                                                    70 
 71         if (!hctx || !(hctx->flags & BLK_MQ_F_     71         if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))
 72                 return true;                       72                 return true;
 73         if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hc     73         if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
 74                 return true;                       74                 return true;
 75                                                    75 
 76         /*                                         76         /*
 77          * Don't try dividing an ant               77          * Don't try dividing an ant
 78          */                                        78          */
 79         if (bt->sb.depth == 1)                     79         if (bt->sb.depth == 1)
 80                 return true;                       80                 return true;
 81                                                    81 
 82         users = atomic_read(&hctx->tags->activ     82         users = atomic_read(&hctx->tags->active_queues);
 83         if (!users)                                83         if (!users)
 84                 return true;                       84                 return true;
 85                                                    85 
 86         /*                                         86         /*
 87          * Allow at least some tags                87          * Allow at least some tags
 88          */                                        88          */
 89         depth = max((bt->sb.depth + users - 1)     89         depth = max((bt->sb.depth + users - 1) / users, 4U);
 90         return atomic_read(&hctx->nr_active) <     90         return atomic_read(&hctx->nr_active) < depth;
 91 }                                                  91 }
 92                                                    92 
 93 static int __blk_mq_get_tag(struct blk_mq_allo !!  93 static int __bt_get(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt)
 94                             struct sbitmap_que << 
 95 {                                                  94 {
 96         if (!(data->flags & BLK_MQ_REQ_INTERNA !!  95         if (!hctx_may_queue(hctx, bt))
 97             !hctx_may_queue(data->hctx, bt))   << 
 98                 return -1;                         96                 return -1;
 99         if (data->shallow_depth)               !!  97         return __sbitmap_queue_get(bt);
100                 return __sbitmap_queue_get_sha << 
101         else                                   << 
102                 return __sbitmap_queue_get(bt) << 
103 }                                                  98 }
104                                                    99 
105 unsigned int blk_mq_get_tag(struct blk_mq_allo !! 100 static int bt_get(struct blk_mq_alloc_data *data, struct sbitmap_queue *bt,
                                                   >> 101                   struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags)
106 {                                                 102 {
107         struct blk_mq_tags *tags = blk_mq_tags << 
108         struct sbitmap_queue *bt;              << 
109         struct sbq_wait_state *ws;                103         struct sbq_wait_state *ws;
110         DEFINE_WAIT(wait);                        104         DEFINE_WAIT(wait);
111         unsigned int tag_offset;               << 
112         bool drop_ctx;                         << 
113         int tag;                                  105         int tag;
114                                                   106 
115         if (data->flags & BLK_MQ_REQ_RESERVED) !! 107         tag = __bt_get(hctx, bt);
116                 if (unlikely(!tags->nr_reserve << 
117                         WARN_ON_ONCE(1);       << 
118                         return BLK_MQ_TAG_FAIL << 
119                 }                              << 
120                 bt = &tags->breserved_tags;    << 
121                 tag_offset = 0;                << 
122         } else {                               << 
123                 bt = &tags->bitmap_tags;       << 
124                 tag_offset = tags->nr_reserved << 
125         }                                      << 
126                                                << 
127         tag = __blk_mq_get_tag(data, bt);      << 
128         if (tag != -1)                            108         if (tag != -1)
129                 goto found_tag;                !! 109                 return tag;
130                                                   110 
131         if (data->flags & BLK_MQ_REQ_NOWAIT)      111         if (data->flags & BLK_MQ_REQ_NOWAIT)
132                 return BLK_MQ_TAG_FAIL;        !! 112                 return -1;
133                                                   113 
134         ws = bt_wait_ptr(bt, data->hctx);      !! 114         ws = bt_wait_ptr(bt, hctx);
135         drop_ctx = data->ctx == NULL;          << 
136         do {                                      115         do {
                                                   >> 116                 prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE);
                                                   >> 117 
                                                   >> 118                 tag = __bt_get(hctx, bt);
                                                   >> 119                 if (tag != -1)
                                                   >> 120                         break;
                                                   >> 121 
137                 /*                                122                 /*
138                  * We're out of tags on this h    123                  * We're out of tags on this hardware queue, kick any
139                  * pending IO submits before g    124                  * pending IO submits before going to sleep waiting for
140                  * some to complete.           !! 125                  * some to complete. Note that hctx can be NULL here for
                                                   >> 126                  * reserved tag allocation.
141                  */                               127                  */
142                 blk_mq_run_hw_queue(data->hctx !! 128                 if (hctx)
                                                   >> 129                         blk_mq_run_hw_queue(hctx, false);
143                                                   130 
144                 /*                                131                 /*
145                  * Retry tag allocation after     132                  * Retry tag allocation after running the hardware queue,
146                  * as running the queue may al    133                  * as running the queue may also have found completions.
147                  */                               134                  */
148                 tag = __blk_mq_get_tag(data, b !! 135                 tag = __bt_get(hctx, bt);
149                 if (tag != -1)                 << 
150                         break;                 << 
151                                                << 
152                 prepare_to_wait_exclusive(&ws- << 
153                                                << 
154                                                << 
155                 tag = __blk_mq_get_tag(data, b << 
156                 if (tag != -1)                    136                 if (tag != -1)
157                         break;                    137                         break;
158                                                   138 
159                 if (data->ctx)                 !! 139                 blk_mq_put_ctx(data->ctx);
160                         blk_mq_put_ctx(data->c << 
161                                                   140 
162                 io_schedule();                    141                 io_schedule();
163                                                   142 
164                 data->ctx = blk_mq_get_ctx(dat    143                 data->ctx = blk_mq_get_ctx(data->q);
165                 data->hctx = blk_mq_map_queue(    144                 data->hctx = blk_mq_map_queue(data->q, data->ctx->cpu);
166                 tags = blk_mq_tags_from_data(d !! 145                 if (data->flags & BLK_MQ_REQ_RESERVED) {
167                 if (data->flags & BLK_MQ_REQ_R !! 146                         bt = &data->hctx->tags->breserved_tags;
168                         bt = &tags->breserved_ !! 147                 } else {
169                 else                           !! 148                         hctx = data->hctx;
170                         bt = &tags->bitmap_tag !! 149                         bt = &hctx->tags->bitmap_tags;
171                                                !! 150                 }
172                 finish_wait(&ws->wait, &wait);    151                 finish_wait(&ws->wait, &wait);
173                 ws = bt_wait_ptr(bt, data->hct !! 152                 ws = bt_wait_ptr(bt, hctx);
174         } while (1);                              153         } while (1);
175                                                   154 
176         if (drop_ctx && data->ctx)             << 
177                 blk_mq_put_ctx(data->ctx);     << 
178                                                << 
179         finish_wait(&ws->wait, &wait);            155         finish_wait(&ws->wait, &wait);
                                                   >> 156         return tag;
                                                   >> 157 }
                                                   >> 158 
                                                   >> 159 static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
                                                   >> 160 {
                                                   >> 161         int tag;
180                                                   162 
181 found_tag:                                     !! 163         tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
182         return tag + tag_offset;               !! 164                      data->hctx->tags);
                                                   >> 165         if (tag >= 0)
                                                   >> 166                 return tag + data->hctx->tags->nr_reserved_tags;
                                                   >> 167 
                                                   >> 168         return BLK_MQ_TAG_FAIL;
183 }                                                 169 }
184                                                   170 
185 void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx !! 171 static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
186                     struct blk_mq_ctx *ctx, un << 
187 {                                                 172 {
188         if (!blk_mq_tag_is_reserved(tags, tag) !! 173         int tag;
                                                   >> 174 
                                                   >> 175         if (unlikely(!data->hctx->tags->nr_reserved_tags)) {
                                                   >> 176                 WARN_ON_ONCE(1);
                                                   >> 177                 return BLK_MQ_TAG_FAIL;
                                                   >> 178         }
                                                   >> 179 
                                                   >> 180         tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL,
                                                   >> 181                      data->hctx->tags);
                                                   >> 182         if (tag < 0)
                                                   >> 183                 return BLK_MQ_TAG_FAIL;
                                                   >> 184 
                                                   >> 185         return tag;
                                                   >> 186 }
                                                   >> 187 
                                                   >> 188 unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
                                                   >> 189 {
                                                   >> 190         if (data->flags & BLK_MQ_REQ_RESERVED)
                                                   >> 191                 return __blk_mq_get_reserved_tag(data);
                                                   >> 192         return __blk_mq_get_tag(data);
                                                   >> 193 }
                                                   >> 194 
                                                   >> 195 void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
                                                   >> 196                     unsigned int tag)
                                                   >> 197 {
                                                   >> 198         struct blk_mq_tags *tags = hctx->tags;
                                                   >> 199 
                                                   >> 200         if (tag >= tags->nr_reserved_tags) {
189                 const int real_tag = tag - tag    201                 const int real_tag = tag - tags->nr_reserved_tags;
190                                                   202 
191                 BUG_ON(real_tag >= tags->nr_ta    203                 BUG_ON(real_tag >= tags->nr_tags);
192                 sbitmap_queue_clear(&tags->bit    204                 sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
193         } else {                                  205         } else {
194                 BUG_ON(tag >= tags->nr_reserve    206                 BUG_ON(tag >= tags->nr_reserved_tags);
195                 sbitmap_queue_clear(&tags->bre    207                 sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
196         }                                         208         }
197 }                                                 209 }
198                                                   210 
199 struct bt_iter_data {                             211 struct bt_iter_data {
200         struct blk_mq_hw_ctx *hctx;               212         struct blk_mq_hw_ctx *hctx;
201         busy_iter_fn *fn;                         213         busy_iter_fn *fn;
202         void *data;                               214         void *data;
203         bool reserved;                            215         bool reserved;
204 };                                                216 };
205                                                   217 
206 static bool bt_iter(struct sbitmap *bitmap, un    218 static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
207 {                                                 219 {
208         struct bt_iter_data *iter_data = data;    220         struct bt_iter_data *iter_data = data;
209         struct blk_mq_hw_ctx *hctx = iter_data    221         struct blk_mq_hw_ctx *hctx = iter_data->hctx;
210         struct blk_mq_tags *tags = hctx->tags;    222         struct blk_mq_tags *tags = hctx->tags;
211         bool reserved = iter_data->reserved;      223         bool reserved = iter_data->reserved;
212         struct request *rq;                       224         struct request *rq;
213                                                   225 
214         if (!reserved)                            226         if (!reserved)
215                 bitnr += tags->nr_reserved_tag    227                 bitnr += tags->nr_reserved_tags;
216         rq = tags->rqs[bitnr];                    228         rq = tags->rqs[bitnr];
217                                                   229 
218         /*                                     !! 230         if (rq->q == hctx->queue)
219          * We can hit rq == NULL here, because << 
220          * test and set the bit before assinin << 
221          */                                    << 
222         if (rq && rq->q == hctx->queue)        << 
223                 iter_data->fn(hctx, rq, iter_d    231                 iter_data->fn(hctx, rq, iter_data->data, reserved);
224         return true;                              232         return true;
225 }                                                 233 }
226                                                   234 
227 static void bt_for_each(struct blk_mq_hw_ctx *    235 static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt,
228                         busy_iter_fn *fn, void    236                         busy_iter_fn *fn, void *data, bool reserved)
229 {                                                 237 {
230         struct bt_iter_data iter_data = {         238         struct bt_iter_data iter_data = {
231                 .hctx = hctx,                     239                 .hctx = hctx,
232                 .fn = fn,                         240                 .fn = fn,
233                 .data = data,                     241                 .data = data,
234                 .reserved = reserved,             242                 .reserved = reserved,
235         };                                        243         };
236                                                   244 
237         sbitmap_for_each_set(&bt->sb, bt_iter,    245         sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data);
238 }                                                 246 }
239                                                   247 
240 struct bt_tags_iter_data {                        248 struct bt_tags_iter_data {
241         struct blk_mq_tags *tags;                 249         struct blk_mq_tags *tags;
242         busy_tag_iter_fn *fn;                     250         busy_tag_iter_fn *fn;
243         void *data;                               251         void *data;
244         bool reserved;                            252         bool reserved;
245 };                                                253 };
246                                                   254 
247 static bool bt_tags_iter(struct sbitmap *bitma    255 static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
248 {                                                 256 {
249         struct bt_tags_iter_data *iter_data =     257         struct bt_tags_iter_data *iter_data = data;
250         struct blk_mq_tags *tags = iter_data->    258         struct blk_mq_tags *tags = iter_data->tags;
251         bool reserved = iter_data->reserved;      259         bool reserved = iter_data->reserved;
252         struct request *rq;                       260         struct request *rq;
253                                                   261 
254         if (!reserved)                            262         if (!reserved)
255                 bitnr += tags->nr_reserved_tag    263                 bitnr += tags->nr_reserved_tags;
256                                                << 
257         /*                                     << 
258          * We can hit rq == NULL here, because << 
259          * test and set the bit before assinin << 
260          */                                    << 
261         rq = tags->rqs[bitnr];                    264         rq = tags->rqs[bitnr];
262         if (rq)                                << 
263                 iter_data->fn(rq, iter_data->d << 
264                                                   265 
                                                   >> 266         iter_data->fn(rq, iter_data->data, reserved);
265         return true;                              267         return true;
266 }                                                 268 }
267                                                   269 
268 static void bt_tags_for_each(struct blk_mq_tag    270 static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt,
269                              busy_tag_iter_fn     271                              busy_tag_iter_fn *fn, void *data, bool reserved)
270 {                                                 272 {
271         struct bt_tags_iter_data iter_data = {    273         struct bt_tags_iter_data iter_data = {
272                 .tags = tags,                     274                 .tags = tags,
273                 .fn = fn,                         275                 .fn = fn,
274                 .data = data,                     276                 .data = data,
275                 .reserved = reserved,             277                 .reserved = reserved,
276         };                                        278         };
277                                                   279 
278         if (tags->rqs)                            280         if (tags->rqs)
279                 sbitmap_for_each_set(&bt->sb,     281                 sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
280 }                                                 282 }
281                                                   283 
282 static void blk_mq_all_tag_busy_iter(struct bl    284 static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags,
283                 busy_tag_iter_fn *fn, void *pr    285                 busy_tag_iter_fn *fn, void *priv)
284 {                                                 286 {
285         if (tags->nr_reserved_tags)               287         if (tags->nr_reserved_tags)
286                 bt_tags_for_each(tags, &tags->    288                 bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, true);
287         bt_tags_for_each(tags, &tags->bitmap_t    289         bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false);
288 }                                                 290 }
289                                                   291 
290 void blk_mq_tagset_busy_iter(struct blk_mq_tag    292 void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
291                 busy_tag_iter_fn *fn, void *pr    293                 busy_tag_iter_fn *fn, void *priv)
292 {                                                 294 {
293         int i;                                    295         int i;
294                                                   296 
295         for (i = 0; i < tagset->nr_hw_queues;     297         for (i = 0; i < tagset->nr_hw_queues; i++) {
296                 if (tagset->tags && tagset->ta    298                 if (tagset->tags && tagset->tags[i])
297                         blk_mq_all_tag_busy_it    299                         blk_mq_all_tag_busy_iter(tagset->tags[i], fn, priv);
298         }                                         300         }
299 }                                                 301 }
300 EXPORT_SYMBOL(blk_mq_tagset_busy_iter);           302 EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
301                                                   303 
302 int blk_mq_tagset_iter(struct blk_mq_tag_set * !! 304 int blk_mq_reinit_tagset(struct blk_mq_tag_set *set)
303                          int (fn)(void *, stru << 
304 {                                                 305 {
305         int i, j, ret = 0;                        306         int i, j, ret = 0;
306                                                   307 
307         if (WARN_ON_ONCE(!fn))                 !! 308         if (!set->ops->reinit_request)
308                 goto out;                         309                 goto out;
309                                                   310 
310         for (i = 0; i < set->nr_hw_queues; i++    311         for (i = 0; i < set->nr_hw_queues; i++) {
311                 struct blk_mq_tags *tags = set    312                 struct blk_mq_tags *tags = set->tags[i];
312                                                   313 
313                 if (!tags)                     << 
314                         continue;              << 
315                                                << 
316                 for (j = 0; j < tags->nr_tags;    314                 for (j = 0; j < tags->nr_tags; j++) {
317                         if (!tags->static_rqs[ !! 315                         if (!tags->rqs[j])
318                                 continue;         316                                 continue;
319                                                   317 
320                         ret = fn(data, tags->s !! 318                         ret = set->ops->reinit_request(set->driver_data,
                                                   >> 319                                                 tags->rqs[j]);
321                         if (ret)                  320                         if (ret)
322                                 goto out;         321                                 goto out;
323                 }                                 322                 }
324         }                                         323         }
325                                                   324 
326 out:                                              325 out:
327         return ret;                               326         return ret;
328 }                                                 327 }
329 EXPORT_SYMBOL_GPL(blk_mq_tagset_iter);         !! 328 EXPORT_SYMBOL_GPL(blk_mq_reinit_tagset);
330                                                   329 
331 void blk_mq_queue_tag_busy_iter(struct request    330 void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
332                 void *priv)                       331                 void *priv)
333 {                                                 332 {
334         struct blk_mq_hw_ctx *hctx;               333         struct blk_mq_hw_ctx *hctx;
335         int i;                                    334         int i;
336                                                   335 
337                                                   336 
338         queue_for_each_hw_ctx(q, hctx, i) {       337         queue_for_each_hw_ctx(q, hctx, i) {
339                 struct blk_mq_tags *tags = hct    338                 struct blk_mq_tags *tags = hctx->tags;
340                                                   339 
341                 /*                                340                 /*
342                  * If not software queues are     341                  * If not software queues are currently mapped to this
343                  * hardware queue, there's not    342                  * hardware queue, there's nothing to check
344                  */                               343                  */
345                 if (!blk_mq_hw_queue_mapped(hc    344                 if (!blk_mq_hw_queue_mapped(hctx))
346                         continue;                 345                         continue;
347                                                   346 
348                 if (tags->nr_reserved_tags)       347                 if (tags->nr_reserved_tags)
349                         bt_for_each(hctx, &tag    348                         bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);
350                 bt_for_each(hctx, &tags->bitma    349                 bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);
351         }                                         350         }
352                                                   351 
353 }                                                 352 }
354                                                   353 
                                                   >> 354 static unsigned int bt_unused_tags(const struct sbitmap_queue *bt)
                                                   >> 355 {
                                                   >> 356         return bt->sb.depth - sbitmap_weight(&bt->sb);
                                                   >> 357 }
                                                   >> 358 
355 static int bt_alloc(struct sbitmap_queue *bt,     359 static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth,
356                     bool round_robin, int node    360                     bool round_robin, int node)
357 {                                                 361 {
358         return sbitmap_queue_init_node(bt, dep    362         return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL,
359                                        node);     363                                        node);
360 }                                                 364 }
361                                                   365 
362 static struct blk_mq_tags *blk_mq_init_bitmap_    366 static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
363                                                   367                                                    int node, int alloc_policy)
364 {                                                 368 {
365         unsigned int depth = tags->nr_tags - t    369         unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
366         bool round_robin = alloc_policy == BLK    370         bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
367                                                   371 
368         if (bt_alloc(&tags->bitmap_tags, depth    372         if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))
369                 goto free_tags;                   373                 goto free_tags;
370         if (bt_alloc(&tags->breserved_tags, ta    374         if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,
371                      node))                       375                      node))
372                 goto free_bitmap_tags;            376                 goto free_bitmap_tags;
373                                                   377 
374         return tags;                              378         return tags;
375 free_bitmap_tags:                                 379 free_bitmap_tags:
376         sbitmap_queue_free(&tags->bitmap_tags)    380         sbitmap_queue_free(&tags->bitmap_tags);
377 free_tags:                                        381 free_tags:
378         kfree(tags);                              382         kfree(tags);
379         return NULL;                              383         return NULL;
380 }                                                 384 }
381                                                   385 
382 struct blk_mq_tags *blk_mq_init_tags(unsigned     386 struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
383                                      unsigned     387                                      unsigned int reserved_tags,
384                                      int node,    388                                      int node, int alloc_policy)
385 {                                                 389 {
386         struct blk_mq_tags *tags;                 390         struct blk_mq_tags *tags;
387                                                   391 
388         if (total_tags > BLK_MQ_TAG_MAX) {        392         if (total_tags > BLK_MQ_TAG_MAX) {
389                 pr_err("blk-mq: tag depth too     393                 pr_err("blk-mq: tag depth too large\n");
390                 return NULL;                      394                 return NULL;
391         }                                         395         }
392                                                   396 
393         tags = kzalloc_node(sizeof(*tags), GFP    397         tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
394         if (!tags)                                398         if (!tags)
395                 return NULL;                      399                 return NULL;
396                                                   400 
397         tags->nr_tags = total_tags;               401         tags->nr_tags = total_tags;
398         tags->nr_reserved_tags = reserved_tags    402         tags->nr_reserved_tags = reserved_tags;
399                                                   403 
400         return blk_mq_init_bitmap_tags(tags, n    404         return blk_mq_init_bitmap_tags(tags, node, alloc_policy);
401 }                                                 405 }
402                                                   406 
403 void blk_mq_free_tags(struct blk_mq_tags *tags    407 void blk_mq_free_tags(struct blk_mq_tags *tags)
404 {                                                 408 {
405         sbitmap_queue_free(&tags->bitmap_tags)    409         sbitmap_queue_free(&tags->bitmap_tags);
406         sbitmap_queue_free(&tags->breserved_ta    410         sbitmap_queue_free(&tags->breserved_tags);
407         kfree(tags);                              411         kfree(tags);
408 }                                                 412 }
409                                                   413 
410 int blk_mq_tag_update_depth(struct blk_mq_hw_c !! 414 int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)
411                             struct blk_mq_tags << 
412                             bool can_grow)     << 
413 {                                                 415 {
414         struct blk_mq_tags *tags = *tagsptr;   << 
415                                                << 
416         if (tdepth <= tags->nr_reserved_tags)  << 
417                 return -EINVAL;                << 
418                                                << 
419         tdepth -= tags->nr_reserved_tags;         416         tdepth -= tags->nr_reserved_tags;
                                                   >> 417         if (tdepth > tags->nr_tags)
                                                   >> 418                 return -EINVAL;
420                                                   419 
421         /*                                        420         /*
422          * If we are allowed to grow beyond th !! 421          * Don't need (or can't) update reserved tags here, they remain
423          * a new set of tags before freeing th !! 422          * static and should never need resizing.
424          */                                       423          */
425         if (tdepth > tags->nr_tags) {          !! 424         sbitmap_queue_resize(&tags->bitmap_tags, tdepth);
426                 struct blk_mq_tag_set *set = h << 
427                 struct blk_mq_tags *new;       << 
428                 bool ret;                      << 
429                                                << 
430                 if (!can_grow)                 << 
431                         return -EINVAL;        << 
432                                                << 
433                 /*                             << 
434                  * We need some sort of upper  << 
435                  * no valid use cases should r << 
436                  */                            << 
437                 if (tdepth > 16 * BLKDEV_MAX_R << 
438                         return -EINVAL;        << 
439                                                << 
440                 new = blk_mq_alloc_rq_map(set, << 
441                 if (!new)                      << 
442                         return -ENOMEM;        << 
443                 ret = blk_mq_alloc_rqs(set, ne << 
444                 if (ret) {                     << 
445                         blk_mq_free_rq_map(new << 
446                         return -ENOMEM;        << 
447                 }                              << 
448                                                << 
449                 blk_mq_free_rqs(set, *tagsptr, << 
450                 blk_mq_free_rq_map(*tagsptr);  << 
451                 *tagsptr = new;                << 
452         } else {                               << 
453                 /*                             << 
454                  * Don't need (or can't) updat << 
455                  * remain static and should ne << 
456                  */                            << 
457                 sbitmap_queue_resize(&tags->bi << 
458         }                                      << 
459                                                   425 
                                                   >> 426         blk_mq_tag_wakeup_all(tags, false);
460         return 0;                                 427         return 0;
461 }                                                 428 }
462                                                   429 
463 /**                                               430 /**
464  * blk_mq_unique_tag() - return a tag that is     431  * blk_mq_unique_tag() - return a tag that is unique queue-wide
465  * @rq: request for which to compute a unique     432  * @rq: request for which to compute a unique tag
466  *                                                433  *
467  * The tag field in struct request is unique p    434  * The tag field in struct request is unique per hardware queue but not over
468  * all hardware queues. Hence this function th    435  * all hardware queues. Hence this function that returns a tag with the
469  * hardware context index in the upper bits an    436  * hardware context index in the upper bits and the per hardware queue tag in
470  * the lower bits.                                437  * the lower bits.
471  *                                                438  *
472  * Note: When called for a request that is que    439  * Note: When called for a request that is queued on a non-multiqueue request
473  * queue, the hardware context index is set to    440  * queue, the hardware context index is set to zero.
474  */                                               441  */
475 u32 blk_mq_unique_tag(struct request *rq)         442 u32 blk_mq_unique_tag(struct request *rq)
476 {                                                 443 {
477         struct request_queue *q = rq->q;          444         struct request_queue *q = rq->q;
478         struct blk_mq_hw_ctx *hctx;               445         struct blk_mq_hw_ctx *hctx;
479         int hwq = 0;                              446         int hwq = 0;
480                                                   447 
481         if (q->mq_ops) {                          448         if (q->mq_ops) {
482                 hctx = blk_mq_map_queue(q, rq-    449                 hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
483                 hwq = hctx->queue_num;            450                 hwq = hctx->queue_num;
484         }                                         451         }
485                                                   452 
486         return (hwq << BLK_MQ_UNIQUE_TAG_BITS)    453         return (hwq << BLK_MQ_UNIQUE_TAG_BITS) |
487                 (rq->tag & BLK_MQ_UNIQUE_TAG_M    454                 (rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
488 }                                                 455 }
489 EXPORT_SYMBOL(blk_mq_unique_tag);                 456 EXPORT_SYMBOL(blk_mq_unique_tag);
                                                   >> 457 
                                                   >> 458 ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page)
                                                   >> 459 {
                                                   >> 460         char *orig_page = page;
                                                   >> 461         unsigned int free, res;
                                                   >> 462 
                                                   >> 463         if (!tags)
                                                   >> 464                 return 0;
                                                   >> 465 
                                                   >> 466         page += sprintf(page, "nr_tags=%u, reserved_tags=%u, "
                                                   >> 467                         "bits_per_word=%u\n",
                                                   >> 468                         tags->nr_tags, tags->nr_reserved_tags,
                                                   >> 469                         1U << tags->bitmap_tags.sb.shift);
                                                   >> 470 
                                                   >> 471         free = bt_unused_tags(&tags->bitmap_tags);
                                                   >> 472         res = bt_unused_tags(&tags->breserved_tags);
                                                   >> 473 
                                                   >> 474         page += sprintf(page, "nr_free=%u, nr_reserved=%u\n", free, res);
                                                   >> 475         page += sprintf(page, "active_queues=%u\n", atomic_read(&tags->active_queues));
                                                   >> 476 
                                                   >> 477         return page - orig_page;
                                                   >> 478 }
490                                                   479 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp