|
@@ -3525,3 +3525,64 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
|
|
|
|
|
|
static void cfq_insert_request(struct request_queue *q, struct request *rq)
|
|
|
{
|
|
|
+ struct cfq_data *cfqd = q->elevator->elevator_data;
|
|
|
+ struct cfq_queue *cfqq = RQ_CFQQ(rq);
|
|
|
+
|
|
|
+ cfq_log_cfqq(cfqd, cfqq, "insert_request");
|
|
|
+ cfq_init_prio_data(cfqq, RQ_CIC(rq));
|
|
|
+
|
|
|
+ rq_set_fifo_time(rq, jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]);
|
|
|
+ list_add_tail(&rq->queuelist, &cfqq->fifo);
|
|
|
+ cfq_add_rq_rb(rq);
|
|
|
+ cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,
|
|
|
+ rq->cmd_flags);
|
|
|
+ cfq_rq_enqueued(cfqd, cfqq, rq);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Update hw_tag based on peak queue depth over 50 samples under
|
|
|
+ * sufficient load.
|
|
|
+ */
|
|
|
+static void cfq_update_hw_tag(struct cfq_data *cfqd)
|
|
|
+{
|
|
|
+ struct cfq_queue *cfqq = cfqd->active_queue;
|
|
|
+
|
|
|
+ if (cfqd->rq_in_driver > cfqd->hw_tag_est_depth)
|
|
|
+ cfqd->hw_tag_est_depth = cfqd->rq_in_driver;
|
|
|
+
|
|
|
+ if (cfqd->hw_tag == 1)
|
|
|
+ return;
|
|
|
+
|
|
|
+ if (cfqd->rq_queued <= CFQ_HW_QUEUE_MIN &&
|
|
|
+ cfqd->rq_in_driver <= CFQ_HW_QUEUE_MIN)
|
|
|
+ return;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * If active queue hasn't enough requests and can idle, cfq might not
|
|
|
+ * dispatch sufficient requests to hardware. Don't zero hw_tag in this
|
|
|
+ * case
|
|
|
+ */
|
|
|
+ if (cfqq && cfq_cfqq_idle_window(cfqq) &&
|
|
|
+ cfqq->dispatched + cfqq->queued[0] + cfqq->queued[1] <
|
|
|
+ CFQ_HW_QUEUE_MIN && cfqd->rq_in_driver < CFQ_HW_QUEUE_MIN)
|
|
|
+ return;
|
|
|
+
|
|
|
+ if (cfqd->hw_tag_samples++ < 50)
|
|
|
+ return;
|
|
|
+
|
|
|
+ if (cfqd->hw_tag_est_depth >= CFQ_HW_QUEUE_MIN)
|
|
|
+ cfqd->hw_tag = 1;
|
|
|
+ else
|
|
|
+ cfqd->hw_tag = 0;
|
|
|
+}
|
|
|
+
|
|
|
+static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)
|
|
|
+{
|
|
|
+ struct cfq_io_cq *cic = cfqd->active_cic;
|
|
|
+
|
|
|
+ /* If the queue already has requests, don't wait */
|
|
|
+ if (!RB_EMPTY_ROOT(&cfqq->sort_list))
|
|
|
+ return false;
|
|
|
+
|
|
|
+ /* If there are other queues in the group, don't wait */
|
|
|
+ if (cfqq->cfqg->nr_cfqq > 1)
|