|
@@ -631,3 +631,175 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
|
|
|
INIT_LIST_HEAD(&q->flush_queue[0]);
|
|
|
INIT_LIST_HEAD(&q->flush_queue[1]);
|
|
|
INIT_LIST_HEAD(&q->flush_data_in_flight);
|
|
|
+ INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
|
|
|
+
|
|
|
+ kobject_init(&q->kobj, &blk_queue_ktype);
|
|
|
+
|
|
|
+ mutex_init(&q->sysfs_lock);
|
|
|
+ spin_lock_init(&q->__queue_lock);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * By default initialize queue_lock to internal lock and driver can
|
|
|
+ * override it later if need be.
|
|
|
+ */
|
|
|
+ q->queue_lock = &q->__queue_lock;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * A queue starts its life with bypass turned on to avoid
|
|
|
+ * unnecessary bypass on/off overhead and nasty surprises during
|
|
|
+ * init. The initial bypass will be finished when the queue is
|
|
|
+ * registered by blk_register_queue().
|
|
|
+ */
|
|
|
+ q->bypass_depth = 1;
|
|
|
+ __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
|
|
|
+
|
|
|
+ if (blkcg_init_queue(q))
|
|
|
+ goto fail_id;
|
|
|
+
|
|
|
+ return q;
|
|
|
+
|
|
|
+fail_id:
|
|
|
+ ida_simple_remove(&blk_queue_ida, q->id);
|
|
|
+fail_q:
|
|
|
+ kmem_cache_free(blk_requestq_cachep, q);
|
|
|
+ return NULL;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_alloc_queue_node);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_init_queue - prepare a request queue for use with a block device
|
|
|
+ * @rfn: The function to be called to process requests that have been
|
|
|
+ * placed on the queue.
|
|
|
+ * @lock: Request queue spin lock
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * If a block device wishes to use the standard request handling procedures,
|
|
|
+ * which sorts requests and coalesces adjacent requests, then it must
|
|
|
+ * call blk_init_queue(). The function @rfn will be called when there
|
|
|
+ * are requests on the queue that need to be processed. If the device
|
|
|
+ * supports plugging, then @rfn may not be called immediately when requests
|
|
|
+ * are available on the queue, but may be called at some time later instead.
|
|
|
+ * Plugged queues are generally unplugged when a buffer belonging to one
|
|
|
+ * of the requests on the queue is needed, or due to memory pressure.
|
|
|
+ *
|
|
|
+ * @rfn is not required, or even expected, to remove all requests off the
|
|
|
+ * queue, but only as many as it can handle at a time. If it does leave
|
|
|
+ * requests on the queue, it is responsible for arranging that the requests
|
|
|
+ * get dealt with eventually.
|
|
|
+ *
|
|
|
+ * The queue spin lock must be held while manipulating the requests on the
|
|
|
+ * request queue; this lock will be taken also from interrupt context, so irq
|
|
|
+ * disabling is needed for it.
|
|
|
+ *
|
|
|
+ * Function returns a pointer to the initialized request queue, or %NULL if
|
|
|
+ * it didn't succeed.
|
|
|
+ *
|
|
|
+ * Note:
|
|
|
+ * blk_init_queue() must be paired with a blk_cleanup_queue() call
|
|
|
+ * when the block device is deactivated (such as at module unload).
|
|
|
+ **/
|
|
|
+
|
|
|
+struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
|
|
|
+{
|
|
|
+ return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_init_queue);
|
|
|
+
|
|
|
+struct request_queue *
|
|
|
+blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
|
|
|
+{
|
|
|
+ struct request_queue *uninit_q, *q;
|
|
|
+
|
|
|
+ uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
|
|
|
+ if (!uninit_q)
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ q = blk_init_allocated_queue(uninit_q, rfn, lock);
|
|
|
+ if (!q)
|
|
|
+ blk_cleanup_queue(uninit_q);
|
|
|
+
|
|
|
+ return q;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_init_queue_node);
|
|
|
+
|
|
|
+struct request_queue *
|
|
|
+blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
|
|
|
+ spinlock_t *lock)
|
|
|
+{
|
|
|
+ if (!q)
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ q->request_fn = rfn;
|
|
|
+ q->prep_rq_fn = NULL;
|
|
|
+ q->unprep_rq_fn = NULL;
|
|
|
+ q->queue_flags |= QUEUE_FLAG_DEFAULT;
|
|
|
+
|
|
|
+ /* Override internal queue lock with supplied lock pointer */
|
|
|
+ if (lock)
|
|
|
+ q->queue_lock = lock;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * This also sets hw/phys segments, boundary and size
|
|
|
+ */
|
|
|
+ blk_queue_make_request(q, blk_queue_bio);
|
|
|
+
|
|
|
+ q->sg_reserved_size = INT_MAX;
|
|
|
+
|
|
|
+ /* init elevator */
|
|
|
+ if (elevator_init(q, NULL))
|
|
|
+ return NULL;
|
|
|
+ return q;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_init_allocated_queue);
|
|
|
+
|
|
|
+bool blk_get_queue(struct request_queue *q)
|
|
|
+{
|
|
|
+ if (likely(!blk_queue_dying(q))) {
|
|
|
+ __blk_get_queue(q);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+ return false;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_get_queue);
|
|
|
+
|
|
|
+static inline void blk_free_request(struct request_list *rl, struct request *rq)
|
|
|
+{
|
|
|
+ if (rq->cmd_flags & REQ_ELVPRIV) {
|
|
|
+ elv_put_request(rl->q, rq);
|
|
|
+ if (rq->elv.icq)
|
|
|
+ put_io_context(rq->elv.icq->ioc);
|
|
|
+ }
|
|
|
+
|
|
|
+ mempool_free(rq, rl->rq_pool);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * ioc_batching returns true if the ioc is a valid batching request and
|
|
|
+ * should be given priority access to a request.
|
|
|
+ */
|
|
|
+static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
|
|
|
+{
|
|
|
+ if (!ioc)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Make sure the process is able to allocate at least 1 request
|
|
|
+ * even if the batch times out, otherwise we could theoretically
|
|
|
+ * lose wakeups.
|
|
|
+ */
|
|
|
+ return ioc->nr_batch_requests == q->nr_batching ||
|
|
|
+ (ioc->nr_batch_requests > 0
|
|
|
+ && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
|
|
|
+ * will cause the process to be a "batcher" on all queues in the system. This
|
|
|
+ * is the behaviour we want though - once it gets a wakeup it should be given
|
|
|
+ * a nice run.
|
|
|
+ */
|
|
|
+static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
|