|  | @@ -2728,3 +2728,138 @@ check_group_idle:
 | 
	
		
			
				|  |  |  	}
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  expire:
 | 
	
		
			
				|  |  | +	cfq_slice_expired(cfqd, 0);
 | 
	
		
			
				|  |  | +new_queue:
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Current queue expired. Check if we have to switch to a new
 | 
	
		
			
				|  |  | +	 * service tree
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	if (!new_cfqq)
 | 
	
		
			
				|  |  | +		cfq_choose_cfqg(cfqd);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	cfqq = cfq_set_active_queue(cfqd, new_cfqq);
 | 
	
		
			
				|  |  | +keep_queue:
 | 
	
		
			
				|  |  | +	return cfqq;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	int dispatched = 0;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	while (cfqq->next_rq) {
 | 
	
		
			
				|  |  | +		cfq_dispatch_insert(cfqq->cfqd->queue, cfqq->next_rq);
 | 
	
		
			
				|  |  | +		dispatched++;
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	BUG_ON(!list_empty(&cfqq->fifo));
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/* By default cfqq is not expired if it is empty. Do it explicitly */
 | 
	
		
			
				|  |  | +	__cfq_slice_expired(cfqq->cfqd, cfqq, 0);
 | 
	
		
			
				|  |  | +	return dispatched;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/*
 | 
	
		
			
				|  |  | + * Drain our current requests. Used for barriers and when switching
 | 
	
		
			
				|  |  | + * io schedulers on-the-fly.
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +static int cfq_forced_dispatch(struct cfq_data *cfqd)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	struct cfq_queue *cfqq;
 | 
	
		
			
				|  |  | +	int dispatched = 0;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/* Expire the timeslice of the current active queue first */
 | 
	
		
			
				|  |  | +	cfq_slice_expired(cfqd, 0);
 | 
	
		
			
				|  |  | +	while ((cfqq = cfq_get_next_queue_forced(cfqd)) != NULL) {
 | 
	
		
			
				|  |  | +		__cfq_set_active_queue(cfqd, cfqq);
 | 
	
		
			
				|  |  | +		dispatched += __cfq_forced_dispatch_cfqq(cfqq);
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	BUG_ON(cfqd->busy_queues);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	cfq_log(cfqd, "forced_dispatch=%d", dispatched);
 | 
	
		
			
				|  |  | +	return dispatched;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static inline bool cfq_slice_used_soon(struct cfq_data *cfqd,
 | 
	
		
			
				|  |  | +	struct cfq_queue *cfqq)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	/* the queue hasn't finished any request, can't estimate */
 | 
	
		
			
				|  |  | +	if (cfq_cfqq_slice_new(cfqq))
 | 
	
		
			
				|  |  | +		return true;
 | 
	
		
			
				|  |  | +	if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched,
 | 
	
		
			
				|  |  | +		cfqq->slice_end))
 | 
	
		
			
				|  |  | +		return true;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	return false;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	unsigned int max_dispatch;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Drain async requests before we start sync IO
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	if (cfq_should_idle(cfqd, cfqq) && cfqd->rq_in_flight[BLK_RW_ASYNC])
 | 
	
		
			
				|  |  | +		return false;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * If this is an async queue and we have sync IO in flight, let it wait
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	if (cfqd->rq_in_flight[BLK_RW_SYNC] && !cfq_cfqq_sync(cfqq))
 | 
	
		
			
				|  |  | +		return false;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	max_dispatch = max_t(unsigned int, cfqd->cfq_quantum / 2, 1);
 | 
	
		
			
				|  |  | +	if (cfq_class_idle(cfqq))
 | 
	
		
			
				|  |  | +		max_dispatch = 1;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Does this cfqq already have too much IO in flight?
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	if (cfqq->dispatched >= max_dispatch) {
 | 
	
		
			
				|  |  | +		bool promote_sync = false;
 | 
	
		
			
				|  |  | +		/*
 | 
	
		
			
				|  |  | +		 * idle queue must always only have a single IO in flight
 | 
	
		
			
				|  |  | +		 */
 | 
	
		
			
				|  |  | +		if (cfq_class_idle(cfqq))
 | 
	
		
			
				|  |  | +			return false;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		/*
 | 
	
		
			
				|  |  | +		 * If there is only one sync queue
 | 
	
		
			
				|  |  | +		 * we can ignore async queue here and give the sync
 | 
	
		
			
				|  |  | +		 * queue no dispatch limit. The reason is a sync queue can
 | 
	
		
			
				|  |  | +		 * preempt async queue, limiting the sync queue doesn't make
 | 
	
		
			
				|  |  | +		 * sense. This is useful for aiostress test.
 | 
	
		
			
				|  |  | +		 */
 | 
	
		
			
				|  |  | +		if (cfq_cfqq_sync(cfqq) && cfqd->busy_sync_queues == 1)
 | 
	
		
			
				|  |  | +			promote_sync = true;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		/*
 | 
	
		
			
				|  |  | +		 * We have other queues, don't allow more IO from this one
 | 
	
		
			
				|  |  | +		 */
 | 
	
		
			
				|  |  | +		if (cfqd->busy_queues > 1 && cfq_slice_used_soon(cfqd, cfqq) &&
 | 
	
		
			
				|  |  | +				!promote_sync)
 | 
	
		
			
				|  |  | +			return false;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		/*
 | 
	
		
			
				|  |  | +		 * Sole queue user, no limit
 | 
	
		
			
				|  |  | +		 */
 | 
	
		
			
				|  |  | +		if (cfqd->busy_queues == 1 || promote_sync)
 | 
	
		
			
				|  |  | +			max_dispatch = -1;
 | 
	
		
			
				|  |  | +		else
 | 
	
		
			
				|  |  | +			/*
 | 
	
		
			
				|  |  | +			 * Normally we start throttling cfqq when cfq_quantum/2
 | 
	
		
			
				|  |  | +			 * requests have been dispatched. But we can drive
 | 
	
		
			
				|  |  | +			 * deeper queue depths at the beginning of slice
 | 
	
		
			
				|  |  | +			 * subjected to upper limit of cfq_quantum.
 | 
	
		
			
				|  |  | +			 * */
 | 
	
		
			
				|  |  | +			max_dispatch = cfqd->cfq_quantum;
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Async queues must wait a bit before being allowed dispatch.
 | 
	
		
			
				|  |  | +	 * We also ramp up the dispatch depth gradually for async IO,
 | 
	
		
			
				|  |  | +	 * based on the last sync IO we serviced
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) {
 | 
	
		
			
				|  |  | +		unsigned long last_sync = jiffies - cfqd->last_delayed_sync;
 |