|
@@ -1950,3 +1950,158 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
|
|
|
should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
|
|
|
return -EIO;
|
|
|
|
|
|
+ spin_lock_irqsave(q->queue_lock, flags);
|
|
|
+ if (unlikely(blk_queue_dying(q))) {
|
|
|
+ spin_unlock_irqrestore(q->queue_lock, flags);
|
|
|
+ return -ENODEV;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Submitting request must be dequeued before calling this function
|
|
|
+ * because it will be linked to another request_queue
|
|
|
+ */
|
|
|
+ BUG_ON(blk_queued_rq(rq));
|
|
|
+
|
|
|
+ if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA))
|
|
|
+ where = ELEVATOR_INSERT_FLUSH;
|
|
|
+
|
|
|
+ add_acct_request(q, rq, where);
|
|
|
+ if (where == ELEVATOR_INSERT_FLUSH)
|
|
|
+ __blk_run_queue(q);
|
|
|
+ spin_unlock_irqrestore(q->queue_lock, flags);
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_rq_err_bytes - determine number of bytes till the next failure boundary
|
|
|
+ * @rq: request to examine
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * A request could be merge of IOs which require different failure
|
|
|
+ * handling. This function determines the number of bytes which
|
|
|
+ * can be failed from the beginning of the request without
|
|
|
+ * crossing into area which need to be retried further.
|
|
|
+ *
|
|
|
+ * Return:
|
|
|
+ * The number of bytes to fail.
|
|
|
+ *
|
|
|
+ * Context:
|
|
|
+ * queue_lock must be held.
|
|
|
+ */
|
|
|
+unsigned int blk_rq_err_bytes(const struct request *rq)
|
|
|
+{
|
|
|
+ unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
|
|
|
+ unsigned int bytes = 0;
|
|
|
+ struct bio *bio;
|
|
|
+
|
|
|
+ if (!(rq->cmd_flags & REQ_MIXED_MERGE))
|
|
|
+ return blk_rq_bytes(rq);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Currently the only 'mixing' which can happen is between
|
|
|
+ * different fastfail types. We can safely fail portions
|
|
|
+ * which have all the failfast bits that the first one has -
|
|
|
+ * the ones which are at least as eager to fail as the first
|
|
|
+ * one.
|
|
|
+ */
|
|
|
+ for (bio = rq->bio; bio; bio = bio->bi_next) {
|
|
|
+ if ((bio->bi_rw & ff) != ff)
|
|
|
+ break;
|
|
|
+ bytes += bio->bi_size;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* this could lead to infinite loop */
|
|
|
+ BUG_ON(blk_rq_bytes(rq) && !bytes);
|
|
|
+ return bytes;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
|
|
|
+
|
|
|
+static void blk_account_io_completion(struct request *req, unsigned int bytes)
|
|
|
+{
|
|
|
+ if (blk_do_io_stat(req)) {
|
|
|
+ const int rw = rq_data_dir(req);
|
|
|
+ struct hd_struct *part;
|
|
|
+ int cpu;
|
|
|
+
|
|
|
+ cpu = part_stat_lock();
|
|
|
+ part = req->part;
|
|
|
+ part_stat_add(cpu, part, sectors[rw], bytes >> 9);
|
|
|
+ part_stat_unlock();
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static void blk_account_io_done(struct request *req)
|
|
|
+{
|
|
|
+ /*
|
|
|
+ * Account IO completion. flush_rq isn't accounted as a
|
|
|
+ * normal IO on queueing nor completion. Accounting the
|
|
|
+ * containing request is enough.
|
|
|
+ */
|
|
|
+ if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {
|
|
|
+ unsigned long duration = jiffies - req->start_time;
|
|
|
+ const int rw = rq_data_dir(req);
|
|
|
+ struct hd_struct *part;
|
|
|
+ int cpu;
|
|
|
+
|
|
|
+ cpu = part_stat_lock();
|
|
|
+ part = req->part;
|
|
|
+
|
|
|
+ part_stat_inc(cpu, part, ios[rw]);
|
|
|
+ part_stat_add(cpu, part, ticks[rw], duration);
|
|
|
+ part_round_stats(cpu, part);
|
|
|
+ part_dec_in_flight(part, rw);
|
|
|
+
|
|
|
+ hd_struct_put(part);
|
|
|
+ part_stat_unlock();
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_peek_request - peek at the top of a request queue
|
|
|
+ * @q: request queue to peek at
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Return the request at the top of @q. The returned request
|
|
|
+ * should be started using blk_start_request() before LLD starts
|
|
|
+ * processing it.
|
|
|
+ *
|
|
|
+ * Return:
|
|
|
+ * Pointer to the request at the top of @q if available. Null
|
|
|
+ * otherwise.
|
|
|
+ *
|
|
|
+ * Context:
|
|
|
+ * queue_lock must be held.
|
|
|
+ */
|
|
|
+struct request *blk_peek_request(struct request_queue *q)
|
|
|
+{
|
|
|
+ struct request *rq;
|
|
|
+ int ret;
|
|
|
+
|
|
|
+ while ((rq = __elv_next_request(q)) != NULL) {
|
|
|
+ if (!(rq->cmd_flags & REQ_STARTED)) {
|
|
|
+ /*
|
|
|
+ * This is the first time the device driver
|
|
|
+ * sees this request (possibly after
|
|
|
+ * requeueing). Notify IO scheduler.
|
|
|
+ */
|
|
|
+ if (rq->cmd_flags & REQ_SORTED)
|
|
|
+ elv_activate_rq(q, rq);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * just mark as started even if we don't start
|
|
|
+ * it, a request that has been delayed should
|
|
|
+ * not be passed by new incoming requests
|
|
|
+ */
|
|
|
+ rq->cmd_flags |= REQ_STARTED;
|
|
|
+ trace_block_rq_issue(q, rq);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!q->boundary_rq || q->boundary_rq == rq) {
|
|
|
+ q->end_sector = rq_end_sector(rq);
|
|
|
+ q->boundary_rq = NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (rq->cmd_flags & REQ_DONTPREP)
|
|
|
+ break;
|