Introduce struct sched_dispatch_ctx, and split the helper into
elevator_dispatch_one_request() and elevator_finish_dispatch(). Also
and comments about the non-error return value.
+struct sched_dispatch_ctx {
+ struct blk_mq_hw_ctx *hctx;
+ struct elevator_queue *e;
+ struct request_queue *q;
+static bool elevator_can_dispatch(struct sched_dispatch_ctx *ctx)
+{
+ if (ctx->e->type->ops.has_work &&
+ !ctx->e->type->ops.has_work(ctx->hctx))
+ return false;
- if (!list_empty_careful(&hctx->dispatch)) {
- busy = true;
- break;
- }
+ if (!list_empty_careful(&ctx->hctx->dispatch)) {
+ ctx->busy = true;
+ return false;
+ }
- budget_token = blk_mq_get_dispatch_budget(q);
- if (budget_token < 0)
- break;
+ return true;
+}
+ if (sq_sched)
+ spin_lock_irq(&ctx->e->lock);
+ rq = ctx->e->type->ops.dispatch_request(ctx->hctx);
+ if (sq_sched)
+ spin_unlock_irq(&ctx->e->lock);
+ if (!rq) {
+ blk_mq_put_dispatch_budget(ctx->q, budget_token);
/*
- * If we cannot get tag for the request, stop dequeueing
- * requests from the IO scheduler. We are unlikely to be able
- * to submit them anyway and it creates false impression for
- * scheduling heuristics that the device can take more IO.
+ * We're releasing without dispatching. Holding the
+ * budget could have blocked any "hctx"s with the
+ * same queue and if we didn't dispatch then there's
+ * no guarantee anyone will kick the queue. Kick it
+ * ourselves.
*/
+static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
+{
+ unsigned int max_dispatch;
+ struct sched_dispatch_ctx ctx = {
+ .hctx = hctx,
+ .q = hctx->queue,
+ .e = hctx->queue->elevator,
+ };
+
+ INIT_LIST_HEAD(&ctx.rq_list);