|
@@ -531,3 +531,111 @@ static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)
|
|
|
|
+{
|
|
|
|
+ struct cfqg_stats *stats = &cfqg->stats;
|
|
|
|
+
|
|
|
|
+ BUG_ON(cfqg_stats_idling(stats));
|
|
|
|
+
|
|
|
|
+ stats->start_idle_time = sched_clock();
|
|
|
|
+ cfqg_stats_mark_idling(stats);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg)
|
|
|
|
+{
|
|
|
|
+ struct cfqg_stats *stats = &cfqg->stats;
|
|
|
|
+
|
|
|
|
+ blkg_stat_add(&stats->avg_queue_size_sum,
|
|
|
|
+ blkg_rwstat_sum(&stats->queued));
|
|
|
|
+ blkg_stat_add(&stats->avg_queue_size_samples, 1);
|
|
|
|
+ cfqg_stats_update_group_wait_time(stats);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+#else /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
|
|
|
|
+
|
|
|
|
+static inline void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg, struct cfq_group *curr_cfqg) { }
|
|
|
|
+static inline void cfqg_stats_end_empty_time(struct cfqg_stats *stats) { }
|
|
|
|
+static inline void cfqg_stats_update_dequeue(struct cfq_group *cfqg) { }
|
|
|
|
+static inline void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg) { }
|
|
|
|
+static inline void cfqg_stats_update_idle_time(struct cfq_group *cfqg) { }
|
|
|
|
+static inline void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg) { }
|
|
|
|
+static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
|
|
|
|
+
|
|
|
|
+#endif /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
|
|
|
|
+
|
|
|
|
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
|
|
|
|
+
|
|
|
|
+static struct blkcg_policy blkcg_policy_cfq;
|
|
|
|
+
|
|
|
|
+static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
|
|
|
|
+{
|
|
|
|
+ return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfqg_get(struct cfq_group *cfqg)
|
|
|
|
+{
|
|
|
|
+ return blkg_get(cfqg_to_blkg(cfqg));
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfqg_put(struct cfq_group *cfqg)
|
|
|
|
+{
|
|
|
|
+ return blkg_put(cfqg_to_blkg(cfqg));
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+#define cfq_log_cfqq(cfqd, cfqq, fmt, args...) do { \
|
|
|
|
+ char __pbuf[128]; \
|
|
|
|
+ \
|
|
|
|
+ blkg_path(cfqg_to_blkg((cfqq)->cfqg), __pbuf, sizeof(__pbuf)); \
|
|
|
|
+ blk_add_trace_msg((cfqd)->queue, "cfq%d%c %s " fmt, (cfqq)->pid, \
|
|
|
|
+ cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
|
|
|
|
+ __pbuf, ##args); \
|
|
|
|
+} while (0)
|
|
|
|
+
|
|
|
|
+#define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do { \
|
|
|
|
+ char __pbuf[128]; \
|
|
|
|
+ \
|
|
|
|
+ blkg_path(cfqg_to_blkg(cfqg), __pbuf, sizeof(__pbuf)); \
|
|
|
|
+ blk_add_trace_msg((cfqd)->queue, "%s " fmt, __pbuf, ##args); \
|
|
|
|
+} while (0)
|
|
|
|
+
|
|
|
|
+static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
|
|
|
|
+ struct cfq_group *curr_cfqg, int rw)
|
|
|
|
+{
|
|
|
|
+ blkg_rwstat_add(&cfqg->stats.queued, rw, 1);
|
|
|
|
+ cfqg_stats_end_empty_time(&cfqg->stats);
|
|
|
|
+ cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
|
|
|
|
+ unsigned long time, unsigned long unaccounted_time)
|
|
|
|
+{
|
|
|
|
+ blkg_stat_add(&cfqg->stats.time, time);
|
|
|
|
+#ifdef CONFIG_DEBUG_BLK_CGROUP
|
|
|
|
+ blkg_stat_add(&cfqg->stats.unaccounted_time, unaccounted_time);
|
|
|
|
+#endif
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw)
|
|
|
|
+{
|
|
|
|
+ blkg_rwstat_add(&cfqg->stats.queued, rw, -1);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw)
|
|
|
|
+{
|
|
|
|
+ blkg_rwstat_add(&cfqg->stats.merged, rw, 1);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg,
|
|
|
|
+ uint64_t bytes, int rw)
|
|
|
|
+{
|
|
|
|
+ blkg_stat_add(&cfqg->stats.sectors, bytes >> 9);
|
|
|
|
+ blkg_rwstat_add(&cfqg->stats.serviced, rw, 1);
|
|
|
|
+ blkg_rwstat_add(&cfqg->stats.service_bytes, rw, bytes);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
|
|
|
|
+ uint64_t start_time, uint64_t io_start_time, int rw)
|
|
|
|
+{
|
|
|
|
+ struct cfqg_stats *stats = &cfqg->stats;
|
|
|
|
+ unsigned long long now = sched_clock();
|
|
|
|
+
|