|
@@ -1318,3 +1318,166 @@ EXPORT_SYMBOL(blk_put_request);
|
|
|
* a block driver. The driver needs to take care of freeing the payload
|
|
|
* itself.
|
|
|
*
|
|
|
+ * Note that this is a quite horrible hack and nothing but handling of
|
|
|
+ * discard requests should ever use it.
|
|
|
+ */
|
|
|
+void blk_add_request_payload(struct request *rq, struct page *page,
|
|
|
+ unsigned int len)
|
|
|
+{
|
|
|
+ struct bio *bio = rq->bio;
|
|
|
+
|
|
|
+ bio->bi_io_vec->bv_page = page;
|
|
|
+ bio->bi_io_vec->bv_offset = 0;
|
|
|
+ bio->bi_io_vec->bv_len = len;
|
|
|
+
|
|
|
+ bio->bi_size = len;
|
|
|
+ bio->bi_vcnt = 1;
|
|
|
+ bio->bi_phys_segments = 1;
|
|
|
+
|
|
|
+ rq->__data_len = rq->resid_len = len;
|
|
|
+ rq->nr_phys_segments = 1;
|
|
|
+ rq->buffer = bio_data(bio);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(blk_add_request_payload);
|
|
|
+
|
|
|
+static bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
|
|
|
+ struct bio *bio)
|
|
|
+{
|
|
|
+ const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
|
|
|
+
|
|
|
+ if (!ll_back_merge_fn(q, req, bio))
|
|
|
+ return false;
|
|
|
+
|
|
|
+ trace_block_bio_backmerge(q, bio);
|
|
|
+
|
|
|
+ if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
|
|
|
+ blk_rq_set_mixed_merge(req);
|
|
|
+
|
|
|
+ req->biotail->bi_next = bio;
|
|
|
+ req->biotail = bio;
|
|
|
+ req->__data_len += bio->bi_size;
|
|
|
+ req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
|
|
|
+
|
|
|
+ drive_stat_acct(req, 0);
|
|
|
+ return true;
|
|
|
+}
|
|
|
+
|
|
|
+static bool bio_attempt_front_merge(struct request_queue *q,
|
|
|
+ struct request *req, struct bio *bio)
|
|
|
+{
|
|
|
+ const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
|
|
|
+
|
|
|
+ if (!ll_front_merge_fn(q, req, bio))
|
|
|
+ return false;
|
|
|
+
|
|
|
+ trace_block_bio_frontmerge(q, bio);
|
|
|
+
|
|
|
+ if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
|
|
|
+ blk_rq_set_mixed_merge(req);
|
|
|
+
|
|
|
+ bio->bi_next = req->bio;
|
|
|
+ req->bio = bio;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * may not be valid. if the low level driver said
|
|
|
+ * it didn't need a bounce buffer then it better
|
|
|
+ * not touch req->buffer either...
|
|
|
+ */
|
|
|
+ req->buffer = bio_data(bio);
|
|
|
+ req->__sector = bio->bi_sector;
|
|
|
+ req->__data_len += bio->bi_size;
|
|
|
+ req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
|
|
|
+
|
|
|
+ drive_stat_acct(req, 0);
|
|
|
+ return true;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * attempt_plug_merge - try to merge with %current's plugged list
|
|
|
+ * @q: request_queue new bio is being queued at
|
|
|
+ * @bio: new bio being queued
|
|
|
+ * @request_count: out parameter for number of traversed plugged requests
|
|
|
+ *
|
|
|
+ * Determine whether @bio being queued on @q can be merged with a request
|
|
|
+ * on %current's plugged list. Returns %true if merge was successful,
|
|
|
+ * otherwise %false.
|
|
|
+ *
|
|
|
+ * Plugging coalesces IOs from the same issuer for the same purpose without
|
|
|
+ * going through @q->queue_lock. As such it's more of an issuing mechanism
|
|
|
+ * than scheduling, and the request, while may have elvpriv data, is not
|
|
|
+ * added on the elevator at this point. In addition, we don't have
|
|
|
+ * reliable access to the elevator outside queue lock. Only check basic
|
|
|
+ * merging parameters without querying the elevator.
|
|
|
+ */
|
|
|
+static bool attempt_plug_merge(struct request_queue *q, struct bio *bio,
|
|
|
+ unsigned int *request_count)
|
|
|
+{
|
|
|
+ struct blk_plug *plug;
|
|
|
+ struct request *rq;
|
|
|
+ bool ret = false;
|
|
|
+
|
|
|
+ plug = current->plug;
|
|
|
+ if (!plug)
|
|
|
+ goto out;
|
|
|
+ *request_count = 0;
|
|
|
+
|
|
|
+ list_for_each_entry_reverse(rq, &plug->list, queuelist) {
|
|
|
+ int el_ret;
|
|
|
+
|
|
|
+ if (rq->q == q)
|
|
|
+ (*request_count)++;
|
|
|
+
|
|
|
+ if (rq->q != q || !blk_rq_merge_ok(rq, bio))
|
|
|
+ continue;
|
|
|
+
|
|
|
+ el_ret = blk_try_merge(rq, bio);
|
|
|
+ if (el_ret == ELEVATOR_BACK_MERGE) {
|
|
|
+ ret = bio_attempt_back_merge(q, rq, bio);
|
|
|
+ if (ret)
|
|
|
+ break;
|
|
|
+ } else if (el_ret == ELEVATOR_FRONT_MERGE) {
|
|
|
+ ret = bio_attempt_front_merge(q, rq, bio);
|
|
|
+ if (ret)
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+out:
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+
|
|
|
+void init_request_from_bio(struct request *req, struct bio *bio)
|
|
|
+{
|
|
|
+ req->cmd_type = REQ_TYPE_FS;
|
|
|
+
|
|
|
+ req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;
|
|
|
+ if (bio->bi_rw & REQ_RAHEAD)
|
|
|
+ req->cmd_flags |= REQ_FAILFAST_MASK;
|
|
|
+
|
|
|
+ req->errors = 0;
|
|
|
+ req->__sector = bio->bi_sector;
|
|
|
+ req->ioprio = bio_prio(bio);
|
|
|
+ blk_rq_bio_prep(req->q, req, bio);
|
|
|
+}
|
|
|
+
|
|
|
+void blk_queue_bio(struct request_queue *q, struct bio *bio)
|
|
|
+{
|
|
|
+ const bool sync = !!(bio->bi_rw & REQ_SYNC);
|
|
|
+ struct blk_plug *plug;
|
|
|
+ int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT;
|
|
|
+ struct request *req;
|
|
|
+ unsigned int request_count = 0;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * low level driver can indicate that it wants pages above a
|
|
|
+ * certain limit bounced to low memory (ie for highmem, or even
|
|
|
+ * ISA dma in theory)
|
|
|
+ */
|
|
|
+ blk_queue_bounce(q, &bio);
|
|
|
+
|
|
|
+ if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ where = ELEVATOR_INSERT_FLUSH;
|
|
|
+ goto get_rq;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|