|
@@ -713,3 +713,146 @@ static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd,
|
|
|
if (group_idle)
|
|
|
slice = cfqd->cfq_group_idle;
|
|
|
else
|
|
|
+ slice = cfqd->cfq_slice_idle;
|
|
|
+ return ttime->ttime_mean > slice;
|
|
|
+}
|
|
|
+
|
|
|
+static inline bool iops_mode(struct cfq_data *cfqd)
|
|
|
+{
|
|
|
+ /*
|
|
|
+ * If we are not idling on queues and it is a NCQ drive, parallel
|
|
|
+ * execution of requests is on and measuring time is not possible
|
|
|
+ * in most of the cases until and unless we drive shallower queue
|
|
|
+ * depths and that becomes a performance bottleneck. In such cases
|
|
|
+ * switch to start providing fairness in terms of number of IOs.
|
|
|
+ */
|
|
|
+ if (!cfqd->cfq_slice_idle && cfqd->hw_tag)
|
|
|
+ return true;
|
|
|
+ else
|
|
|
+ return false;
|
|
|
+}
|
|
|
+
|
|
|
+static inline enum wl_prio_t cfqq_prio(struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ if (cfq_class_idle(cfqq))
|
|
|
+ return IDLE_WORKLOAD;
|
|
|
+ if (cfq_class_rt(cfqq))
|
|
|
+ return RT_WORKLOAD;
|
|
|
+ return BE_WORKLOAD;
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+static enum wl_type_t cfqq_type(struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ if (!cfq_cfqq_sync(cfqq))
|
|
|
+ return ASYNC_WORKLOAD;
|
|
|
+ if (!cfq_cfqq_idle_window(cfqq))
|
|
|
+ return SYNC_NOIDLE_WORKLOAD;
|
|
|
+ return SYNC_WORKLOAD;
|
|
|
+}
|
|
|
+
|
|
|
+static inline int cfq_group_busy_queues_wl(enum wl_prio_t wl,
|
|
|
+ struct cfq_data *cfqd,
|
|
|
+ struct cfq_group *cfqg)
|
|
|
+{
|
|
|
+ if (wl == IDLE_WORKLOAD)
|
|
|
+ return cfqg->service_tree_idle.count;
|
|
|
+
|
|
|
+ return cfqg->service_trees[wl][ASYNC_WORKLOAD].count
|
|
|
+ + cfqg->service_trees[wl][SYNC_NOIDLE_WORKLOAD].count
|
|
|
+ + cfqg->service_trees[wl][SYNC_WORKLOAD].count;
|
|
|
+}
|
|
|
+
|
|
|
+static inline int cfqg_busy_async_queues(struct cfq_data *cfqd,
|
|
|
+ struct cfq_group *cfqg)
|
|
|
+{
|
|
|
+ return cfqg->service_trees[RT_WORKLOAD][ASYNC_WORKLOAD].count
|
|
|
+ + cfqg->service_trees[BE_WORKLOAD][ASYNC_WORKLOAD].count;
|
|
|
+}
|
|
|
+
|
|
|
+static void cfq_dispatch_insert(struct request_queue *, struct request *);
|
|
|
+static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync,
|
|
|
+ struct cfq_io_cq *cic, struct bio *bio,
|
|
|
+ gfp_t gfp_mask);
|
|
|
+
|
|
|
+static inline struct cfq_io_cq *icq_to_cic(struct io_cq *icq)
|
|
|
+{
|
|
|
+ /* cic->icq is the first member, %NULL will convert to %NULL */
|
|
|
+ return container_of(icq, struct cfq_io_cq, icq);
|
|
|
+}
|
|
|
+
|
|
|
+static inline struct cfq_io_cq *cfq_cic_lookup(struct cfq_data *cfqd,
|
|
|
+ struct io_context *ioc)
|
|
|
+{
|
|
|
+ if (ioc)
|
|
|
+ return icq_to_cic(ioc_lookup_icq(ioc, cfqd->queue));
|
|
|
+ return NULL;
|
|
|
+}
|
|
|
+
|
|
|
+static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)
|
|
|
+{
|
|
|
+ return cic->cfqq[is_sync];
|
|
|
+}
|
|
|
+
|
|
|
+static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,
|
|
|
+ bool is_sync)
|
|
|
+{
|
|
|
+ cic->cfqq[is_sync] = cfqq;
|
|
|
+}
|
|
|
+
|
|
|
+static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic)
|
|
|
+{
|
|
|
+ return cic->icq.q->elevator->elevator_data;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * We regard a request as SYNC, if it's either a read or has the SYNC bit
|
|
|
+ * set (in which case it could also be direct WRITE).
|
|
|
+ */
|
|
|
+static inline bool cfq_bio_sync(struct bio *bio)
|
|
|
+{
|
|
|
+ return bio_data_dir(bio) == READ || (bio->bi_rw & REQ_SYNC);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * scheduler run of queue, if there are requests pending and no one in the
|
|
|
+ * driver that will restart queueing
|
|
|
+ */
|
|
|
+static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
|
|
|
+{
|
|
|
+ if (cfqd->busy_queues) {
|
|
|
+ cfq_log(cfqd, "schedule dispatch");
|
|
|
+ kblockd_schedule_work(cfqd->queue, &cfqd->unplug_work);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Scale schedule slice based on io priority. Use the sync time slice only
|
|
|
+ * if a queue is marked sync and has sync io queued. A sync queue with async
|
|
|
+ * io only, should not get full sync slice length.
|
|
|
+ */
|
|
|
+static inline int cfq_prio_slice(struct cfq_data *cfqd, bool sync,
|
|
|
+ unsigned short prio)
|
|
|
+{
|
|
|
+ const int base_slice = cfqd->cfq_slice[sync];
|
|
|
+
|
|
|
+ WARN_ON(prio >= IOPRIO_BE_NR);
|
|
|
+
|
|
|
+ return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - prio));
|
|
|
+}
|
|
|
+
|
|
|
+static inline int
|
|
|
+cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio);
|
|
|
+}
|
|
|
+
|
|
|
+static inline u64 cfq_scale_slice(unsigned long delta, struct cfq_group *cfqg)
|
|
|
+{
|
|
|
+ u64 d = delta << CFQ_SERVICE_SHIFT;
|
|
|
+
|
|
|
+ d = d * CFQ_WEIGHT_DEFAULT;
|
|
|
+ do_div(d, cfqg->weight);
|
|
|
+ return d;
|
|
|
+}
|
|
|
+
|