|
@@ -365,3 +365,91 @@ static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
|
|
* synchroniztion and takes care of u64 atomicity.
|
|
* synchroniztion and takes care of u64 atomicity.
|
|
*/
|
|
*/
|
|
static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
|
|
static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
|
|
|
|
+{
|
|
|
|
+ unsigned int start;
|
|
|
|
+ uint64_t v;
|
|
|
|
+
|
|
|
|
+ do {
|
|
|
|
+ start = u64_stats_fetch_begin(&stat->syncp);
|
|
|
|
+ v = stat->cnt;
|
|
|
|
+ } while (u64_stats_fetch_retry(&stat->syncp, start));
|
|
|
|
+
|
|
|
|
+ return v;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * blkg_stat_reset - reset a blkg_stat
|
|
|
|
+ * @stat: blkg_stat to reset
|
|
|
|
+ */
|
|
|
|
+static inline void blkg_stat_reset(struct blkg_stat *stat)
|
|
|
|
+{
|
|
|
|
+ stat->cnt = 0;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * blkg_rwstat_add - add a value to a blkg_rwstat
|
|
|
|
+ * @rwstat: target blkg_rwstat
|
|
|
|
+ * @rw: mask of REQ_{WRITE|SYNC}
|
|
|
|
+ * @val: value to add
|
|
|
|
+ *
|
|
|
|
+ * Add @val to @rwstat. The counters are chosen according to @rw. The
|
|
|
|
+ * caller is responsible for synchronizing calls to this function.
|
|
|
|
+ */
|
|
|
|
+static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
|
|
|
|
+ int rw, uint64_t val)
|
|
|
|
+{
|
|
|
|
+ u64_stats_update_begin(&rwstat->syncp);
|
|
|
|
+
|
|
|
|
+ if (rw & REQ_WRITE)
|
|
|
|
+ rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
|
|
|
|
+ else
|
|
|
|
+ rwstat->cnt[BLKG_RWSTAT_READ] += val;
|
|
|
|
+ if (rw & REQ_SYNC)
|
|
|
|
+ rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
|
|
|
|
+ else
|
|
|
|
+ rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;
|
|
|
|
+
|
|
|
|
+ u64_stats_update_end(&rwstat->syncp);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * blkg_rwstat_read - read the current values of a blkg_rwstat
|
|
|
|
+ * @rwstat: blkg_rwstat to read
|
|
|
|
+ *
|
|
|
|
+ * Read the current snapshot of @rwstat and return it as the return value.
|
|
|
|
+ * This function can be called without synchronization and takes care of
|
|
|
|
+ * u64 atomicity.
|
|
|
|
+ */
|
|
|
|
+static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
|
|
|
|
+{
|
|
|
|
+ unsigned int start;
|
|
|
|
+ struct blkg_rwstat tmp;
|
|
|
|
+
|
|
|
|
+ do {
|
|
|
|
+ start = u64_stats_fetch_begin(&rwstat->syncp);
|
|
|
|
+ tmp = *rwstat;
|
|
|
|
+ } while (u64_stats_fetch_retry(&rwstat->syncp, start));
|
|
|
|
+
|
|
|
|
+ return tmp;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * blkg_rwstat_sum - read the total count of a blkg_rwstat
|
|
|
|
+ * @rwstat: blkg_rwstat to read
|
|
|
|
+ *
|
|
|
|
+ * Return the total count of @rwstat regardless of the IO direction. This
|
|
|
|
+ * function can be called without synchronization and takes care of u64
|
|
|
|
+ * atomicity.
|
|
|
|
+ */
|
|
|
|
+static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat)
|
|
|
|
+{
|
|
|
|
+ struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
|
|
|
|
+
|
|
|
|
+ return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * blkg_rwstat_reset - reset a blkg_rwstat
|
|
|
|
+ * @rwstat: blkg_rwstat to reset
|
|
|
|
+ */
|
|
|
|
+static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
|