|
@@ -978,3 +978,144 @@ static struct request *__get_request(struct request_list *rl, int rw_flags,
|
|
|
icq = ioc_lookup_icq(ioc, q);
|
|
|
}
|
|
|
|
|
|
+ if (blk_queue_io_stat(q))
|
|
|
+ rw_flags |= REQ_IO_STAT;
|
|
|
+ spin_unlock_irq(q->queue_lock);
|
|
|
+
|
|
|
+ /* allocate and init request */
|
|
|
+ rq = mempool_alloc(rl->rq_pool, gfp_mask);
|
|
|
+ if (!rq)
|
|
|
+ goto fail_alloc;
|
|
|
+
|
|
|
+ blk_rq_init(q, rq);
|
|
|
+ blk_rq_set_rl(rq, rl);
|
|
|
+ rq->cmd_flags = rw_flags | REQ_ALLOCED;
|
|
|
+
|
|
|
+ /* init elvpriv */
|
|
|
+ if (rw_flags & REQ_ELVPRIV) {
|
|
|
+ if (unlikely(et->icq_cache && !icq)) {
|
|
|
+ if (ioc)
|
|
|
+ icq = ioc_create_icq(ioc, q, gfp_mask);
|
|
|
+ if (!icq)
|
|
|
+ goto fail_elvpriv;
|
|
|
+ }
|
|
|
+
|
|
|
+ rq->elv.icq = icq;
|
|
|
+ if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
|
|
|
+ goto fail_elvpriv;
|
|
|
+
|
|
|
+ /* @rq->elv.icq holds io_context until @rq is freed */
|
|
|
+ if (icq)
|
|
|
+ get_io_context(icq->ioc);
|
|
|
+ }
|
|
|
+out:
|
|
|
+ /*
|
|
|
+ * ioc may be NULL here, and ioc_batching will be false. That's
|
|
|
+ * OK, if the queue is under the request limit then requests need
|
|
|
+ * not count toward the nr_batch_requests limit. There will always
|
|
|
+ * be some limit enforced by BLK_BATCH_TIME.
|
|
|
+ */
|
|
|
+ if (ioc_batching(q, ioc))
|
|
|
+ ioc->nr_batch_requests--;
|
|
|
+
|
|
|
+ trace_block_getrq(q, bio, rw_flags & 1);
|
|
|
+ return rq;
|
|
|
+
|
|
|
+fail_elvpriv:
|
|
|
+ /*
|
|
|
+ * elvpriv init failed. ioc, icq and elvpriv aren't mempool backed
|
|
|
+ * and may fail indefinitely under memory pressure and thus
|
|
|
+ * shouldn't stall IO. Treat this request as !elvpriv. This will
|
|
|
+ * disturb iosched and blkcg but weird is bettern than dead.
|
|
|
+ */
|
|
|
+ printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n",
|
|
|
+ dev_name(q->backing_dev_info.dev));
|
|
|
+
|
|
|
+ rq->cmd_flags &= ~REQ_ELVPRIV;
|
|
|
+ rq->elv.icq = NULL;
|
|
|
+
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ q->nr_rqs_elvpriv--;
|
|
|
+ spin_unlock_irq(q->queue_lock);
|
|
|
+ goto out;
|
|
|
+
|
|
|
+fail_alloc:
|
|
|
+ /*
|
|
|
+ * Allocation failed presumably due to memory. Undo anything we
|
|
|
+ * might have messed up.
|
|
|
+ *
|
|
|
+ * Allocating task should really be put onto the front of the wait
|
|
|
+ * queue, but this is pretty rare.
|
|
|
+ */
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ freed_request(rl, rw_flags);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * in the very unlikely event that allocation failed and no
|
|
|
+ * requests for this direction was pending, mark us starved so that
|
|
|
+ * freeing of a request in the other direction will notice
|
|
|
+ * us. another possible fix would be to split the rq mempool into
|
|
|
+ * READ and WRITE
|
|
|
+ */
|
|
|
+rq_starved:
|
|
|
+ if (unlikely(rl->count[is_sync] == 0))
|
|
|
+ rl->starved[is_sync] = 1;
|
|
|
+ return NULL;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * get_request - get a free request
|
|
|
+ * @q: request_queue to allocate request from
|
|
|
+ * @rw_flags: RW and SYNC flags
|
|
|
+ * @bio: bio to allocate request for (can be %NULL)
|
|
|
+ * @gfp_mask: allocation mask
|
|
|
+ *
|
|
|
+ * Get a free request from @q. If %__GFP_WAIT is set in @gfp_mask, this
|
|
|
+ * function keeps retrying under memory pressure and fails iff @q is dead.
|
|
|
+ *
|
|
|
+ * Must be callled with @q->queue_lock held and,
|
|
|
+ * Returns %NULL on failure, with @q->queue_lock held.
|
|
|
+ * Returns !%NULL on success, with @q->queue_lock *not held*.
|
|
|
+ */
|
|
|
+static struct request *get_request(struct request_queue *q, int rw_flags,
|
|
|
+ struct bio *bio, gfp_t gfp_mask)
|
|
|
+{
|
|
|
+ const bool is_sync = rw_is_sync(rw_flags) != 0;
|
|
|
+ DEFINE_WAIT(wait);
|
|
|
+ struct request_list *rl;
|
|
|
+ struct request *rq;
|
|
|
+
|
|
|
+ rl = blk_get_rl(q, bio); /* transferred to @rq on success */
|
|
|
+retry:
|
|
|
+ rq = __get_request(rl, rw_flags, bio, gfp_mask);
|
|
|
+ if (rq)
|
|
|
+ return rq;
|
|
|
+
|
|
|
+ if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) {
|
|
|
+ blk_put_rl(rl);
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* wait on @rl and retry */
|
|
|
+ prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
|
|
|
+ TASK_UNINTERRUPTIBLE);
|
|
|
+
|
|
|
+ trace_block_sleeprq(q, bio, rw_flags & 1);
|
|
|
+
|
|
|
+ spin_unlock_irq(q->queue_lock);
|
|
|
+ io_schedule();
|
|
|
+
|
|
|
+ /*
|
|
|
+ * After sleeping, we become a "batching" process and will be able
|
|
|
+ * to allocate at least one request, and up to a big batch of them
|
|
|
+ * for a small period time. See ioc_batching, ioc_set_batching
|
|
|
+ */
|
|
|
+ ioc_set_batching(q, current->io_context);
|
|
|
+
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ finish_wait(&rl->wait[is_sync], &wait);
|
|
|
+
|
|
|
+ goto retry;
|
|
|
+}
|
|
|
+
|
|
|
+struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
|