|
@@ -900,3 +900,189 @@ out:
|
|
|
|
|
|
/*
|
|
|
* If we dispatched some requests, unplug the queue to make sure
|
|
|
+ * immediate dispatch
|
|
|
+ */
|
|
|
+ if (nr_disp) {
|
|
|
+ blk_start_plug(&plug);
|
|
|
+ while((bio = bio_list_pop(&bio_list_on_stack)))
|
|
|
+ generic_make_request(bio);
|
|
|
+ blk_finish_plug(&plug);
|
|
|
+ }
|
|
|
+ return nr_disp;
|
|
|
+}
|
|
|
+
|
|
|
+void blk_throtl_work(struct work_struct *work)
|
|
|
+{
|
|
|
+ struct throtl_data *td = container_of(work, struct throtl_data,
|
|
|
+ throtl_work.work);
|
|
|
+ struct request_queue *q = td->queue;
|
|
|
+
|
|
|
+ throtl_dispatch(q);
|
|
|
+}
|
|
|
+
|
|
|
+/* Call with queue lock held */
|
|
|
+static void
|
|
|
+throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay)
|
|
|
+{
|
|
|
+
|
|
|
+ struct delayed_work *dwork = &td->throtl_work;
|
|
|
+
|
|
|
+ /* schedule work if limits changed even if no bio is queued */
|
|
|
+ if (total_nr_queued(td) || td->limits_changed) {
|
|
|
+ mod_delayed_work(kthrotld_workqueue, dwork, delay);
|
|
|
+ throtl_log(td, "schedule work. delay=%lu jiffies=%lu",
|
|
|
+ delay, jiffies);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
|
|
|
+ struct blkg_policy_data *pd, int off)
|
|
|
+{
|
|
|
+ struct throtl_grp *tg = pd_to_tg(pd);
|
|
|
+ struct blkg_rwstat rwstat = { }, tmp;
|
|
|
+ int i, cpu;
|
|
|
+
|
|
|
+ for_each_possible_cpu(cpu) {
|
|
|
+ struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
|
|
|
+
|
|
|
+ tmp = blkg_rwstat_read((void *)sc + off);
|
|
|
+ for (i = 0; i < BLKG_RWSTAT_NR; i++)
|
|
|
+ rwstat.cnt[i] += tmp.cnt[i];
|
|
|
+ }
|
|
|
+
|
|
|
+ return __blkg_prfill_rwstat(sf, pd, &rwstat);
|
|
|
+}
|
|
|
+
|
|
|
+static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
|
|
+ struct seq_file *sf)
|
|
|
+{
|
|
|
+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
|
|
|
+
|
|
|
+ blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat, &blkcg_policy_throtl,
|
|
|
+ cft->private, true);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd,
|
|
|
+ int off)
|
|
|
+{
|
|
|
+ struct throtl_grp *tg = pd_to_tg(pd);
|
|
|
+ u64 v = *(u64 *)((void *)tg + off);
|
|
|
+
|
|
|
+ if (v == -1)
|
|
|
+ return 0;
|
|
|
+ return __blkg_prfill_u64(sf, pd, v);
|
|
|
+}
|
|
|
+
|
|
|
+static u64 tg_prfill_conf_uint(struct seq_file *sf, struct blkg_policy_data *pd,
|
|
|
+ int off)
|
|
|
+{
|
|
|
+ struct throtl_grp *tg = pd_to_tg(pd);
|
|
|
+ unsigned int v = *(unsigned int *)((void *)tg + off);
|
|
|
+
|
|
|
+ if (v == -1)
|
|
|
+ return 0;
|
|
|
+ return __blkg_prfill_u64(sf, pd, v);
|
|
|
+}
|
|
|
+
|
|
|
+static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft,
|
|
|
+ struct seq_file *sf)
|
|
|
+{
|
|
|
+ blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_u64,
|
|
|
+ &blkcg_policy_throtl, cft->private, false);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static int tg_print_conf_uint(struct cgroup *cgrp, struct cftype *cft,
|
|
|
+ struct seq_file *sf)
|
|
|
+{
|
|
|
+ blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_uint,
|
|
|
+ &blkcg_policy_throtl, cft->private, false);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
|
|
|
+ bool is_u64)
|
|
|
+{
|
|
|
+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
|
|
|
+ struct blkg_conf_ctx ctx;
|
|
|
+ struct throtl_grp *tg;
|
|
|
+ struct throtl_data *td;
|
|
|
+ int ret;
|
|
|
+
|
|
|
+ ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);
|
|
|
+ if (ret)
|
|
|
+ return ret;
|
|
|
+
|
|
|
+ tg = blkg_to_tg(ctx.blkg);
|
|
|
+ td = ctx.blkg->q->td;
|
|
|
+
|
|
|
+ if (!ctx.v)
|
|
|
+ ctx.v = -1;
|
|
|
+
|
|
|
+ if (is_u64)
|
|
|
+ *(u64 *)((void *)tg + cft->private) = ctx.v;
|
|
|
+ else
|
|
|
+ *(unsigned int *)((void *)tg + cft->private) = ctx.v;
|
|
|
+
|
|
|
+ /* XXX: we don't need the following deferred processing */
|
|
|
+ xchg(&tg->limits_changed, true);
|
|
|
+ xchg(&td->limits_changed, true);
|
|
|
+ throtl_schedule_delayed_work(td, 0);
|
|
|
+
|
|
|
+ blkg_conf_finish(&ctx);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static int tg_set_conf_u64(struct cgroup *cgrp, struct cftype *cft,
|
|
|
+ const char *buf)
|
|
|
+{
|
|
|
+ return tg_set_conf(cgrp, cft, buf, true);
|
|
|
+}
|
|
|
+
|
|
|
+static int tg_set_conf_uint(struct cgroup *cgrp, struct cftype *cft,
|
|
|
+ const char *buf)
|
|
|
+{
|
|
|
+ return tg_set_conf(cgrp, cft, buf, false);
|
|
|
+}
|
|
|
+
|
|
|
+static struct cftype throtl_files[] = {
|
|
|
+ {
|
|
|
+ .name = "throttle.read_bps_device",
|
|
|
+ .private = offsetof(struct throtl_grp, bps[READ]),
|
|
|
+ .read_seq_string = tg_print_conf_u64,
|
|
|
+ .write_string = tg_set_conf_u64,
|
|
|
+ .max_write_len = 256,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ .name = "throttle.write_bps_device",
|
|
|
+ .private = offsetof(struct throtl_grp, bps[WRITE]),
|
|
|
+ .read_seq_string = tg_print_conf_u64,
|
|
|
+ .write_string = tg_set_conf_u64,
|
|
|
+ .max_write_len = 256,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ .name = "throttle.read_iops_device",
|
|
|
+ .private = offsetof(struct throtl_grp, iops[READ]),
|
|
|
+ .read_seq_string = tg_print_conf_uint,
|
|
|
+ .write_string = tg_set_conf_uint,
|
|
|
+ .max_write_len = 256,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ .name = "throttle.write_iops_device",
|
|
|
+ .private = offsetof(struct throtl_grp, iops[WRITE]),
|
|
|
+ .read_seq_string = tg_print_conf_uint,
|
|
|
+ .write_string = tg_set_conf_uint,
|
|
|
+ .max_write_len = 256,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ .name = "throttle.io_service_bytes",
|
|
|
+ .private = offsetof(struct tg_stats_cpu, service_bytes),
|
|
|
+ .read_seq_string = tg_print_cpu_rwstat,
|
|
|
+ },
|
|
|
+ {
|
|
|
+ .name = "throttle.io_serviced",
|
|
|
+ .private = offsetof(struct tg_stats_cpu, serviced),
|
|
|
+ .read_seq_string = tg_print_cpu_rwstat,
|
|
|
+ },
|
|
|
+ { } /* terminate */
|