~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/block/blk-rq-qos.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include "blk-rq-qos.h"
  2 
  3 /*
  4  * Increment 'v', if 'v' is below 'below'. Returns true if we succeeded,
  5  * false if 'v' + 1 would be bigger than 'below'.
  6  */
  7 static bool atomic_inc_below(atomic_t *v, unsigned int below)
  8 {
  9         unsigned int cur = atomic_read(v);
 10 
 11         for (;;) {
 12                 unsigned int old;
 13 
 14                 if (cur >= below)
 15                         return false;
 16                 old = atomic_cmpxchg(v, cur, cur + 1);
 17                 if (old == cur)
 18                         break;
 19                 cur = old;
 20         }
 21 
 22         return true;
 23 }
 24 
 25 bool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit)
 26 {
 27         return atomic_inc_below(&rq_wait->inflight, limit);
 28 }
 29 
 30 void __rq_qos_cleanup(struct rq_qos *rqos, struct bio *bio)
 31 {
 32         do {
 33                 if (rqos->ops->cleanup)
 34                         rqos->ops->cleanup(rqos, bio);
 35                 rqos = rqos->next;
 36         } while (rqos);
 37 }
 38 
 39 void __rq_qos_done(struct rq_qos *rqos, struct request *rq)
 40 {
 41         do {
 42                 if (rqos->ops->done)
 43                         rqos->ops->done(rqos, rq);
 44                 rqos = rqos->next;
 45         } while (rqos);
 46 }
 47 
 48 void __rq_qos_issue(struct rq_qos *rqos, struct request *rq)
 49 {
 50         do {
 51                 if (rqos->ops->issue)
 52                         rqos->ops->issue(rqos, rq);
 53                 rqos = rqos->next;
 54         } while (rqos);
 55 }
 56 
 57 void __rq_qos_requeue(struct rq_qos *rqos, struct request *rq)
 58 {
 59         do {
 60                 if (rqos->ops->requeue)
 61                         rqos->ops->requeue(rqos, rq);
 62                 rqos = rqos->next;
 63         } while (rqos);
 64 }
 65 
 66 void __rq_qos_throttle(struct rq_qos *rqos, struct bio *bio)
 67 {
 68         do {
 69                 if (rqos->ops->throttle)
 70                         rqos->ops->throttle(rqos, bio);
 71                 rqos = rqos->next;
 72         } while (rqos);
 73 }
 74 
 75 void __rq_qos_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
 76 {
 77         do {
 78                 if (rqos->ops->track)
 79                         rqos->ops->track(rqos, rq, bio);
 80                 rqos = rqos->next;
 81         } while (rqos);
 82 }
 83 
 84 void __rq_qos_done_bio(struct rq_qos *rqos, struct bio *bio)
 85 {
 86         do {
 87                 if (rqos->ops->done_bio)
 88                         rqos->ops->done_bio(rqos, bio);
 89                 rqos = rqos->next;
 90         } while (rqos);
 91 }
 92 
 93 /*
 94  * Return true, if we can't increase the depth further by scaling
 95  */
 96 bool rq_depth_calc_max_depth(struct rq_depth *rqd)
 97 {
 98         unsigned int depth;
 99         bool ret = false;
100 
101         /*
102          * For QD=1 devices, this is a special case. It's important for those
103          * to have one request ready when one completes, so force a depth of
104          * 2 for those devices. On the backend, it'll be a depth of 1 anyway,
105          * since the device can't have more than that in flight. If we're
106          * scaling down, then keep a setting of 1/1/1.
107          */
108         if (rqd->queue_depth == 1) {
109                 if (rqd->scale_step > 0)
110                         rqd->max_depth = 1;
111                 else {
112                         rqd->max_depth = 2;
113                         ret = true;
114                 }
115         } else {
116                 /*
117                  * scale_step == 0 is our default state. If we have suffered
118                  * latency spikes, step will be > 0, and we shrink the
119                  * allowed write depths. If step is < 0, we're only doing
120                  * writes, and we allow a temporarily higher depth to
121                  * increase performance.
122                  */
123                 depth = min_t(unsigned int, rqd->default_depth,
124                               rqd->queue_depth);
125                 if (rqd->scale_step > 0)
126                         depth = 1 + ((depth - 1) >> min(31, rqd->scale_step));
127                 else if (rqd->scale_step < 0) {
128                         unsigned int maxd = 3 * rqd->queue_depth / 4;
129 
130                         depth = 1 + ((depth - 1) << -rqd->scale_step);
131                         if (depth > maxd) {
132                                 depth = maxd;
133                                 ret = true;
134                         }
135                 }
136 
137                 rqd->max_depth = depth;
138         }
139 
140         return ret;
141 }
142 
143 void rq_depth_scale_up(struct rq_depth *rqd)
144 {
145         /*
146          * Hit max in previous round, stop here
147          */
148         if (rqd->scaled_max)
149                 return;
150 
151         rqd->scale_step--;
152 
153         rqd->scaled_max = rq_depth_calc_max_depth(rqd);
154 }
155 
156 /*
157  * Scale rwb down. If 'hard_throttle' is set, do it quicker, since we
158  * had a latency violation.
159  */
160 void rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle)
161 {
162         /*
163          * Stop scaling down when we've hit the limit. This also prevents
164          * ->scale_step from going to crazy values, if the device can't
165          * keep up.
166          */
167         if (rqd->max_depth == 1)
168                 return;
169 
170         if (rqd->scale_step < 0 && hard_throttle)
171                 rqd->scale_step = 0;
172         else
173                 rqd->scale_step++;
174 
175         rqd->scaled_max = false;
176         rq_depth_calc_max_depth(rqd);
177 }
178 
179 struct rq_qos_wait_data {
180         struct wait_queue_entry wq;
181         struct task_struct *task;
182         struct rq_wait *rqw;
183         acquire_inflight_cb_t *cb;
184         void *private_data;
185         bool got_token;
186 };
187 
188 static int rq_qos_wake_function(struct wait_queue_entry *curr,
189                                 unsigned int mode, int wake_flags, void *key)
190 {
191         struct rq_qos_wait_data *data = container_of(curr,
192                                                      struct rq_qos_wait_data,
193                                                      wq);
194 
195         /*
196          * If we fail to get a budget, return -1 to interrupt the wake up loop
197          * in __wake_up_common.
198          */
199         if (!data->cb(data->rqw, data->private_data))
200                 return -1;
201 
202         data->got_token = true;
203         list_del_init(&curr->entry);
204         wake_up_process(data->task);
205         return 1;
206 }
207 
208 /**
209  * rq_qos_wait - throttle on a rqw if we need to
210  * @private_data - caller provided specific data
211  * @acquire_inflight_cb - inc the rqw->inflight counter if we can
212  * @cleanup_cb - the callback to cleanup in case we race with a waker
213  *
214  * This provides a uniform place for the rq_qos users to do their throttling.
215  * Since you can end up with a lot of things sleeping at once, this manages the
216  * waking up based on the resources available.  The acquire_inflight_cb should
217  * inc the rqw->inflight if we have the ability to do so, or return false if not
218  * and then we will sleep until the room becomes available.
219  *
220  * cleanup_cb is in case that we race with a waker and need to cleanup the
221  * inflight count accordingly.
222  */
223 void rq_qos_wait(struct rq_wait *rqw, void *private_data,
224                  acquire_inflight_cb_t *acquire_inflight_cb,
225                  cleanup_cb_t *cleanup_cb)
226 {
227         struct rq_qos_wait_data data = {
228                 .wq = {
229                         .func   = rq_qos_wake_function,
230                         .entry  = LIST_HEAD_INIT(data.wq.entry),
231                 },
232                 .task = current,
233                 .rqw = rqw,
234                 .cb = acquire_inflight_cb,
235                 .private_data = private_data,
236         };
237         bool has_sleeper;
238 
239         has_sleeper = wq_has_sleeper(&rqw->wait);
240         if (!has_sleeper && acquire_inflight_cb(rqw, private_data))
241                 return;
242 
243         prepare_to_wait_exclusive(&rqw->wait, &data.wq, TASK_UNINTERRUPTIBLE);
244         do {
245                 if (data.got_token)
246                         break;
247                 if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) {
248                         finish_wait(&rqw->wait, &data.wq);
249 
250                         /*
251                          * We raced with wbt_wake_function() getting a token,
252                          * which means we now have two. Put our local token
253                          * and wake anyone else potentially waiting for one.
254                          */
255                         if (data.got_token)
256                                 cleanup_cb(rqw, private_data);
257                         break;
258                 }
259                 io_schedule();
260                 has_sleeper = false;
261         } while (1);
262         finish_wait(&rqw->wait, &data.wq);
263 }
264 
265 void rq_qos_exit(struct request_queue *q)
266 {
267         blk_mq_debugfs_unregister_queue_rqos(q);
268 
269         while (q->rq_qos) {
270                 struct rq_qos *rqos = q->rq_qos;
271                 q->rq_qos = rqos->next;
272                 rqos->ops->exit(rqos);
273         }
274 }
275 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp