|  | @@ -631,3 +631,175 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
 | 
	
		
			
				|  |  |  	INIT_LIST_HEAD(&q->flush_queue[0]);
 | 
	
		
			
				|  |  |  	INIT_LIST_HEAD(&q->flush_queue[1]);
 | 
	
		
			
				|  |  |  	INIT_LIST_HEAD(&q->flush_data_in_flight);
 | 
	
		
			
				|  |  | +	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	kobject_init(&q->kobj, &blk_queue_ktype);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	mutex_init(&q->sysfs_lock);
 | 
	
		
			
				|  |  | +	spin_lock_init(&q->__queue_lock);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * By default initialize queue_lock to internal lock and driver can
 | 
	
		
			
				|  |  | +	 * override it later if need be.
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	q->queue_lock = &q->__queue_lock;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * A queue starts its life with bypass turned on to avoid
 | 
	
		
			
				|  |  | +	 * unnecessary bypass on/off overhead and nasty surprises during
 | 
	
		
			
				|  |  | +	 * init.  The initial bypass will be finished when the queue is
 | 
	
		
			
				|  |  | +	 * registered by blk_register_queue().
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	q->bypass_depth = 1;
 | 
	
		
			
				|  |  | +	__set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if (blkcg_init_queue(q))
 | 
	
		
			
				|  |  | +		goto fail_id;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	return q;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +fail_id:
 | 
	
		
			
				|  |  | +	ida_simple_remove(&blk_queue_ida, q->id);
 | 
	
		
			
				|  |  | +fail_q:
 | 
	
		
			
				|  |  | +	kmem_cache_free(blk_requestq_cachep, q);
 | 
	
		
			
				|  |  | +	return NULL;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL(blk_alloc_queue_node);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/**
 | 
	
		
			
				|  |  | + * blk_init_queue  - prepare a request queue for use with a block device
 | 
	
		
			
				|  |  | + * @rfn:  The function to be called to process requests that have been
 | 
	
		
			
				|  |  | + *        placed on the queue.
 | 
	
		
			
				|  |  | + * @lock: Request queue spin lock
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Description:
 | 
	
		
			
				|  |  | + *    If a block device wishes to use the standard request handling procedures,
 | 
	
		
			
				|  |  | + *    which sorts requests and coalesces adjacent requests, then it must
 | 
	
		
			
				|  |  | + *    call blk_init_queue().  The function @rfn will be called when there
 | 
	
		
			
				|  |  | + *    are requests on the queue that need to be processed.  If the device
 | 
	
		
			
				|  |  | + *    supports plugging, then @rfn may not be called immediately when requests
 | 
	
		
			
				|  |  | + *    are available on the queue, but may be called at some time later instead.
 | 
	
		
			
				|  |  | + *    Plugged queues are generally unplugged when a buffer belonging to one
 | 
	
		
			
				|  |  | + *    of the requests on the queue is needed, or due to memory pressure.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + *    @rfn is not required, or even expected, to remove all requests off the
 | 
	
		
			
				|  |  | + *    queue, but only as many as it can handle at a time.  If it does leave
 | 
	
		
			
				|  |  | + *    requests on the queue, it is responsible for arranging that the requests
 | 
	
		
			
				|  |  | + *    get dealt with eventually.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + *    The queue spin lock must be held while manipulating the requests on the
 | 
	
		
			
				|  |  | + *    request queue; this lock will be taken also from interrupt context, so irq
 | 
	
		
			
				|  |  | + *    disabling is needed for it.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + *    Function returns a pointer to the initialized request queue, or %NULL if
 | 
	
		
			
				|  |  | + *    it didn't succeed.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Note:
 | 
	
		
			
				|  |  | + *    blk_init_queue() must be paired with a blk_cleanup_queue() call
 | 
	
		
			
				|  |  | + *    when the block device is deactivated (such as at module unload).
 | 
	
		
			
				|  |  | + **/
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL(blk_init_queue);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +struct request_queue *
 | 
	
		
			
				|  |  | +blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	struct request_queue *uninit_q, *q;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
 | 
	
		
			
				|  |  | +	if (!uninit_q)
 | 
	
		
			
				|  |  | +		return NULL;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	q = blk_init_allocated_queue(uninit_q, rfn, lock);
 | 
	
		
			
				|  |  | +	if (!q)
 | 
	
		
			
				|  |  | +		blk_cleanup_queue(uninit_q);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	return q;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL(blk_init_queue_node);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +struct request_queue *
 | 
	
		
			
				|  |  | +blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
 | 
	
		
			
				|  |  | +			 spinlock_t *lock)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	if (!q)
 | 
	
		
			
				|  |  | +		return NULL;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
 | 
	
		
			
				|  |  | +		return NULL;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	q->request_fn		= rfn;
 | 
	
		
			
				|  |  | +	q->prep_rq_fn		= NULL;
 | 
	
		
			
				|  |  | +	q->unprep_rq_fn		= NULL;
 | 
	
		
			
				|  |  | +	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/* Override internal queue lock with supplied lock pointer */
 | 
	
		
			
				|  |  | +	if (lock)
 | 
	
		
			
				|  |  | +		q->queue_lock		= lock;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * This also sets hw/phys segments, boundary and size
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	blk_queue_make_request(q, blk_queue_bio);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	q->sg_reserved_size = INT_MAX;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/* init elevator */
 | 
	
		
			
				|  |  | +	if (elevator_init(q, NULL))
 | 
	
		
			
				|  |  | +		return NULL;
 | 
	
		
			
				|  |  | +	return q;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL(blk_init_allocated_queue);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +bool blk_get_queue(struct request_queue *q)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	if (likely(!blk_queue_dying(q))) {
 | 
	
		
			
				|  |  | +		__blk_get_queue(q);
 | 
	
		
			
				|  |  | +		return true;
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	return false;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL(blk_get_queue);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static inline void blk_free_request(struct request_list *rl, struct request *rq)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	if (rq->cmd_flags & REQ_ELVPRIV) {
 | 
	
		
			
				|  |  | +		elv_put_request(rl->q, rq);
 | 
	
		
			
				|  |  | +		if (rq->elv.icq)
 | 
	
		
			
				|  |  | +			put_io_context(rq->elv.icq->ioc);
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	mempool_free(rq, rl->rq_pool);
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/*
 | 
	
		
			
				|  |  | + * ioc_batching returns true if the ioc is a valid batching request and
 | 
	
		
			
				|  |  | + * should be given priority access to a request.
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	if (!ioc)
 | 
	
		
			
				|  |  | +		return 0;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Make sure the process is able to allocate at least 1 request
 | 
	
		
			
				|  |  | +	 * even if the batch times out, otherwise we could theoretically
 | 
	
		
			
				|  |  | +	 * lose wakeups.
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	return ioc->nr_batch_requests == q->nr_batching ||
 | 
	
		
			
				|  |  | +		(ioc->nr_batch_requests > 0
 | 
	
		
			
				|  |  | +		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/*
 | 
	
		
			
				|  |  | + * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
 | 
	
		
			
				|  |  | + * will cause the process to be a "batcher" on all queues in the system. This
 | 
	
		
			
				|  |  | + * is the behaviour we want though - once it gets a wakeup it should be given
 | 
	
		
			
				|  |  | + * a nice run.
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
 |