|
@@ -2074,3 +2074,131 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
|
|
* queues apart again.
|
|
* queues apart again.
|
|
*/
|
|
*/
|
|
if (cfq_cfqq_coop(cfqq) && CFQQ_SEEKY(cfqq))
|
|
if (cfq_cfqq_coop(cfqq) && CFQQ_SEEKY(cfqq))
|
|
|
|
+ cfq_mark_cfqq_split_coop(cfqq);
|
|
|
|
+
|
|
|
|
+ /*
|
|
|
|
+ * store what was left of this slice, if the queue idled/timed out
|
|
|
|
+ */
|
|
|
|
+ if (timed_out) {
|
|
|
|
+ if (cfq_cfqq_slice_new(cfqq))
|
|
|
|
+ cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq);
|
|
|
|
+ else
|
|
|
|
+ cfqq->slice_resid = cfqq->slice_end - jiffies;
|
|
|
|
+ cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid);
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ cfq_group_served(cfqd, cfqq->cfqg, cfqq);
|
|
|
|
+
|
|
|
|
+ if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list))
|
|
|
|
+ cfq_del_cfqq_rr(cfqd, cfqq);
|
|
|
|
+
|
|
|
|
+ cfq_resort_rr_list(cfqd, cfqq);
|
|
|
|
+
|
|
|
|
+ if (cfqq == cfqd->active_queue)
|
|
|
|
+ cfqd->active_queue = NULL;
|
|
|
|
+
|
|
|
|
+ if (cfqd->active_cic) {
|
|
|
|
+ put_io_context(cfqd->active_cic->icq.ioc);
|
|
|
|
+ cfqd->active_cic = NULL;
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfq_slice_expired(struct cfq_data *cfqd, bool timed_out)
|
|
|
|
+{
|
|
|
|
+ struct cfq_queue *cfqq = cfqd->active_queue;
|
|
|
|
+
|
|
|
|
+ if (cfqq)
|
|
|
|
+ __cfq_slice_expired(cfqd, cfqq, timed_out);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/*
|
|
|
|
+ * Get next queue for service. Unless we have a queue preemption,
|
|
|
|
+ * we'll simply select the first cfqq in the service tree.
|
|
|
|
+ */
|
|
|
|
+static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd)
|
|
|
|
+{
|
|
|
|
+ struct cfq_rb_root *service_tree =
|
|
|
|
+ service_tree_for(cfqd->serving_group, cfqd->serving_prio,
|
|
|
|
+ cfqd->serving_type);
|
|
|
|
+
|
|
|
|
+ if (!cfqd->rq_queued)
|
|
|
|
+ return NULL;
|
|
|
|
+
|
|
|
|
+ /* There is nothing to dispatch */
|
|
|
|
+ if (!service_tree)
|
|
|
|
+ return NULL;
|
|
|
|
+ if (RB_EMPTY_ROOT(&service_tree->rb))
|
|
|
|
+ return NULL;
|
|
|
|
+ return cfq_rb_first(service_tree);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static struct cfq_queue *cfq_get_next_queue_forced(struct cfq_data *cfqd)
|
|
|
|
+{
|
|
|
|
+ struct cfq_group *cfqg;
|
|
|
|
+ struct cfq_queue *cfqq;
|
|
|
|
+ int i, j;
|
|
|
|
+ struct cfq_rb_root *st;
|
|
|
|
+
|
|
|
|
+ if (!cfqd->rq_queued)
|
|
|
|
+ return NULL;
|
|
|
|
+
|
|
|
|
+ cfqg = cfq_get_next_cfqg(cfqd);
|
|
|
|
+ if (!cfqg)
|
|
|
|
+ return NULL;
|
|
|
|
+
|
|
|
|
+ for_each_cfqg_st(cfqg, i, j, st)
|
|
|
|
+ if ((cfqq = cfq_rb_first(st)) != NULL)
|
|
|
|
+ return cfqq;
|
|
|
|
+ return NULL;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/*
|
|
|
|
+ * Get and set a new active queue for service.
|
|
|
|
+ */
|
|
|
|
+static struct cfq_queue *cfq_set_active_queue(struct cfq_data *cfqd,
|
|
|
|
+ struct cfq_queue *cfqq)
|
|
|
|
+{
|
|
|
|
+ if (!cfqq)
|
|
|
|
+ cfqq = cfq_get_next_queue(cfqd);
|
|
|
|
+
|
|
|
|
+ __cfq_set_active_queue(cfqd, cfqq);
|
|
|
|
+ return cfqq;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline sector_t cfq_dist_from_last(struct cfq_data *cfqd,
|
|
|
|
+ struct request *rq)
|
|
|
|
+{
|
|
|
|
+ if (blk_rq_pos(rq) >= cfqd->last_position)
|
|
|
|
+ return blk_rq_pos(rq) - cfqd->last_position;
|
|
|
|
+ else
|
|
|
|
+ return cfqd->last_position - blk_rq_pos(rq);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq,
|
|
|
|
+ struct request *rq)
|
|
|
|
+{
|
|
|
|
+ return cfq_dist_from_last(cfqd, rq) <= CFQQ_CLOSE_THR;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
|
|
|
|
+ struct cfq_queue *cur_cfqq)
|
|
|
|
+{
|
|
|
|
+ struct rb_root *root = &cfqd->prio_trees[cur_cfqq->org_ioprio];
|
|
|
|
+ struct rb_node *parent, *node;
|
|
|
|
+ struct cfq_queue *__cfqq;
|
|
|
|
+ sector_t sector = cfqd->last_position;
|
|
|
|
+
|
|
|
|
+ if (RB_EMPTY_ROOT(root))
|
|
|
|
+ return NULL;
|
|
|
|
+
|
|
|
|
+ /*
|
|
|
|
+ * First, if we find a request starting at the end of the last
|
|
|
|
+ * request, choose it.
|
|
|
|
+ */
|
|
|
|
+ __cfqq = cfq_prio_tree_lookup(cfqd, root, sector, &parent, NULL);
|
|
|
|
+ if (__cfqq)
|
|
|
|
+ return __cfqq;
|
|
|
|
+
|
|
|
|
+ /*
|
|
|
|
+ * If the exact sector wasn't found, the parent of the NULL leaf
|
|
|
|
+ * will contain the closest sector.
|