|
|
@@ -1664,3 +1664,134 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
|
|
|
|
|
|
parent = *p;
|
|
|
__cfqq = rb_entry(parent, struct cfq_queue, rb_node);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * sort by key, that represents service time.
|
|
|
+ */
|
|
|
+ if (time_before(rb_key, __cfqq->rb_key))
|
|
|
+ n = &(*p)->rb_left;
|
|
|
+ else {
|
|
|
+ n = &(*p)->rb_right;
|
|
|
+ left = 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ p = n;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (left)
|
|
|
+ service_tree->left = &cfqq->rb_node;
|
|
|
+
|
|
|
+ cfqq->rb_key = rb_key;
|
|
|
+ rb_link_node(&cfqq->rb_node, parent, p);
|
|
|
+ rb_insert_color(&cfqq->rb_node, &service_tree->rb);
|
|
|
+ service_tree->count++;
|
|
|
+ if (add_front || !new_cfqq)
|
|
|
+ return;
|
|
|
+ cfq_group_notify_queue_add(cfqd, cfqq->cfqg);
|
|
|
+}
|
|
|
+
|
|
|
+static struct cfq_queue *
|
|
|
+cfq_prio_tree_lookup(struct cfq_data *cfqd, struct rb_root *root,
|
|
|
+ sector_t sector, struct rb_node **ret_parent,
|
|
|
+ struct rb_node ***rb_link)
|
|
|
+{
|
|
|
+ struct rb_node **p, *parent;
|
|
|
+ struct cfq_queue *cfqq = NULL;
|
|
|
+
|
|
|
+ parent = NULL;
|
|
|
+ p = &root->rb_node;
|
|
|
+ while (*p) {
|
|
|
+ struct rb_node **n;
|
|
|
+
|
|
|
+ parent = *p;
|
|
|
+ cfqq = rb_entry(parent, struct cfq_queue, p_node);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Sort strictly based on sector. Smallest to the left,
|
|
|
+ * largest to the right.
|
|
|
+ */
|
|
|
+ if (sector > blk_rq_pos(cfqq->next_rq))
|
|
|
+ n = &(*p)->rb_right;
|
|
|
+ else if (sector < blk_rq_pos(cfqq->next_rq))
|
|
|
+ n = &(*p)->rb_left;
|
|
|
+ else
|
|
|
+ break;
|
|
|
+ p = n;
|
|
|
+ cfqq = NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ *ret_parent = parent;
|
|
|
+ if (rb_link)
|
|
|
+ *rb_link = p;
|
|
|
+ return cfqq;
|
|
|
+}
|
|
|
+
|
|
|
+static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ struct rb_node **p, *parent;
|
|
|
+ struct cfq_queue *__cfqq;
|
|
|
+
|
|
|
+ if (cfqq->p_root) {
|
|
|
+ rb_erase(&cfqq->p_node, cfqq->p_root);
|
|
|
+ cfqq->p_root = NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (cfq_class_idle(cfqq))
|
|
|
+ return;
|
|
|
+ if (!cfqq->next_rq)
|
|
|
+ return;
|
|
|
+
|
|
|
+ cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio];
|
|
|
+ __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root,
|
|
|
+ blk_rq_pos(cfqq->next_rq), &parent, &p);
|
|
|
+ if (!__cfqq) {
|
|
|
+ rb_link_node(&cfqq->p_node, parent, p);
|
|
|
+ rb_insert_color(&cfqq->p_node, cfqq->p_root);
|
|
|
+ } else
|
|
|
+ cfqq->p_root = NULL;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Update cfqq's position in the service tree.
|
|
|
+ */
|
|
|
+static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ /*
|
|
|
+ * Resorting requires the cfqq to be on the RR list already.
|
|
|
+ */
|
|
|
+ if (cfq_cfqq_on_rr(cfqq)) {
|
|
|
+ cfq_service_tree_add(cfqd, cfqq, 0);
|
|
|
+ cfq_prio_tree_add(cfqd, cfqq);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * add to busy list of queues for service, trying to be fair in ordering
|
|
|
+ * the pending list according to last request service
|
|
|
+ */
|
|
|
+static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ cfq_log_cfqq(cfqd, cfqq, "add_to_rr");
|
|
|
+ BUG_ON(cfq_cfqq_on_rr(cfqq));
|
|
|
+ cfq_mark_cfqq_on_rr(cfqq);
|
|
|
+ cfqd->busy_queues++;
|
|
|
+ if (cfq_cfqq_sync(cfqq))
|
|
|
+ cfqd->busy_sync_queues++;
|
|
|
+
|
|
|
+ cfq_resort_rr_list(cfqd, cfqq);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Called when the cfqq no longer has requests pending, remove it from
|
|
|
+ * the service tree.
|
|
|
+ */
|
|
|
+static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ cfq_log_cfqq(cfqd, cfqq, "del_from_rr");
|
|
|
+ BUG_ON(!cfq_cfqq_on_rr(cfqq));
|
|
|
+ cfq_clear_cfqq_on_rr(cfqq);
|
|
|
+
|
|
|
+ if (!RB_EMPTY_NODE(&cfqq->rb_node)) {
|
|
|
+ cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree);
|
|
|
+ cfqq->service_tree = NULL;
|
|
|
+ }
|
