| 
					
				 | 
			
			
				@@ -713,3 +713,146 @@ static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 	if (group_idle) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 		slice = cfqd->cfq_group_idle; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 	else 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		slice = cfqd->cfq_slice_idle; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return ttime->ttime_mean > slice; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline bool iops_mode(struct cfq_data *cfqd) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * If we are not idling on queues and it is a NCQ drive, parallel 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * execution of requests is on and measuring time is not possible 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * in most of the cases until and unless we drive shallower queue 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * depths and that becomes a performance bottleneck. In such cases 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * switch to start providing fairness in terms of number of IOs. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (!cfqd->cfq_slice_idle && cfqd->hw_tag) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return true; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	else 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return false; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline enum wl_prio_t cfqq_prio(struct cfq_queue *cfqq) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (cfq_class_idle(cfqq)) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return IDLE_WORKLOAD; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (cfq_class_rt(cfqq)) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return RT_WORKLOAD; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return BE_WORKLOAD; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static enum wl_type_t cfqq_type(struct cfq_queue *cfqq) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (!cfq_cfqq_sync(cfqq)) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return ASYNC_WORKLOAD; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (!cfq_cfqq_idle_window(cfqq)) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return SYNC_NOIDLE_WORKLOAD; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return SYNC_WORKLOAD; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline int cfq_group_busy_queues_wl(enum wl_prio_t wl, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+					struct cfq_data *cfqd, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+					struct cfq_group *cfqg) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (wl == IDLE_WORKLOAD) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return cfqg->service_tree_idle.count; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return cfqg->service_trees[wl][ASYNC_WORKLOAD].count 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		+ cfqg->service_trees[wl][SYNC_NOIDLE_WORKLOAD].count 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		+ cfqg->service_trees[wl][SYNC_WORKLOAD].count; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline int cfqg_busy_async_queues(struct cfq_data *cfqd, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+					struct cfq_group *cfqg) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return cfqg->service_trees[RT_WORKLOAD][ASYNC_WORKLOAD].count 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		+ cfqg->service_trees[BE_WORKLOAD][ASYNC_WORKLOAD].count; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static void cfq_dispatch_insert(struct request_queue *, struct request *); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+				       struct cfq_io_cq *cic, struct bio *bio, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+				       gfp_t gfp_mask); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline struct cfq_io_cq *icq_to_cic(struct io_cq *icq) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* cic->icq is the first member, %NULL will convert to %NULL */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return container_of(icq, struct cfq_io_cq, icq); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline struct cfq_io_cq *cfq_cic_lookup(struct cfq_data *cfqd, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+					       struct io_context *ioc) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (ioc) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return icq_to_cic(ioc_lookup_icq(ioc, cfqd->queue)); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return NULL; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return cic->cfqq[is_sync]; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+				bool is_sync) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	cic->cfqq[is_sync] = cfqq; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return cic->icq.q->elevator->elevator_data; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * We regard a request as SYNC, if it's either a read or has the SYNC bit 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * set (in which case it could also be direct WRITE). 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline bool cfq_bio_sync(struct bio *bio) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return bio_data_dir(bio) == READ || (bio->bi_rw & REQ_SYNC); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * scheduler run of queue, if there are requests pending and no one in the 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * driver that will restart queueing 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline void cfq_schedule_dispatch(struct cfq_data *cfqd) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (cfqd->busy_queues) { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		cfq_log(cfqd, "schedule dispatch"); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		kblockd_schedule_work(cfqd->queue, &cfqd->unplug_work); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * Scale schedule slice based on io priority. Use the sync time slice only 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * if a queue is marked sync and has sync io queued. A sync queue with async 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * io only, should not get full sync slice length. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline int cfq_prio_slice(struct cfq_data *cfqd, bool sync, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+				 unsigned short prio) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	const int base_slice = cfqd->cfq_slice[sync]; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	WARN_ON(prio >= IOPRIO_BE_NR); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - prio)); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline int 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static inline u64 cfq_scale_slice(unsigned long delta, struct cfq_group *cfqg) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	u64 d = delta << CFQ_SERVICE_SHIFT; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	d = d * CFQ_WEIGHT_DEFAULT; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	do_div(d, cfqg->weight); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return d; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 |