|
@@ -190,3 +190,199 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
|
|
|
|
|
|
if (q->dma_drain_size && q->dma_drain_needed(rq)) {
|
|
if (q->dma_drain_size && q->dma_drain_needed(rq)) {
|
|
if (rq->cmd_flags & REQ_WRITE)
|
|
if (rq->cmd_flags & REQ_WRITE)
|
|
|
|
+ memset(q->dma_drain_buffer, 0, q->dma_drain_size);
|
|
|
|
+
|
|
|
|
+ sg->page_link &= ~0x02;
|
|
|
|
+ sg = sg_next(sg);
|
|
|
|
+ sg_set_page(sg, virt_to_page(q->dma_drain_buffer),
|
|
|
|
+ q->dma_drain_size,
|
|
|
|
+ ((unsigned long)q->dma_drain_buffer) &
|
|
|
|
+ (PAGE_SIZE - 1));
|
|
|
|
+ nsegs++;
|
|
|
|
+ rq->extra_len += q->dma_drain_size;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (sg)
|
|
|
|
+ sg_mark_end(sg);
|
|
|
|
+
|
|
|
|
+ return nsegs;
|
|
|
|
+}
|
|
|
|
+EXPORT_SYMBOL(blk_rq_map_sg);
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * blk_bio_map_sg - map a bio to a scatterlist
|
|
|
|
+ * @q: request_queue in question
|
|
|
|
+ * @bio: bio being mapped
|
|
|
|
+ * @sglist: scatterlist being mapped
|
|
|
|
+ *
|
|
|
|
+ * Note:
|
|
|
|
+ * Caller must make sure sg can hold bio->bi_phys_segments entries
|
|
|
|
+ *
|
|
|
|
+ * Will return the number of sg entries setup
|
|
|
|
+ */
|
|
|
|
+int blk_bio_map_sg(struct request_queue *q, struct bio *bio,
|
|
|
|
+ struct scatterlist *sglist)
|
|
|
|
+{
|
|
|
|
+ struct bio_vec *bvec, *bvprv;
|
|
|
|
+ struct scatterlist *sg;
|
|
|
|
+ int nsegs, cluster;
|
|
|
|
+ unsigned long i;
|
|
|
|
+
|
|
|
|
+ nsegs = 0;
|
|
|
|
+ cluster = blk_queue_cluster(q);
|
|
|
|
+
|
|
|
|
+ bvprv = NULL;
|
|
|
|
+ sg = NULL;
|
|
|
|
+ bio_for_each_segment(bvec, bio, i) {
|
|
|
|
+ __blk_segment_map_sg(q, bvec, sglist, &bvprv, &sg,
|
|
|
|
+ &nsegs, &cluster);
|
|
|
|
+ } /* segments in bio */
|
|
|
|
+
|
|
|
|
+ if (sg)
|
|
|
|
+ sg_mark_end(sg);
|
|
|
|
+
|
|
|
|
+ BUG_ON(bio->bi_phys_segments && nsegs > bio->bi_phys_segments);
|
|
|
|
+ return nsegs;
|
|
|
|
+}
|
|
|
|
+EXPORT_SYMBOL(blk_bio_map_sg);
|
|
|
|
+
|
|
|
|
+static inline int ll_new_hw_segment(struct request_queue *q,
|
|
|
|
+ struct request *req,
|
|
|
|
+ struct bio *bio)
|
|
|
|
+{
|
|
|
|
+ int nr_phys_segs = bio_phys_segments(q, bio);
|
|
|
|
+
|
|
|
|
+ if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(q))
|
|
|
|
+ goto no_merge;
|
|
|
|
+
|
|
|
|
+ if (bio_integrity(bio) && blk_integrity_merge_bio(q, req, bio))
|
|
|
|
+ goto no_merge;
|
|
|
|
+
|
|
|
|
+ /*
|
|
|
|
+ * This will form the start of a new hw segment. Bump both
|
|
|
|
+ * counters.
|
|
|
|
+ */
|
|
|
|
+ req->nr_phys_segments += nr_phys_segs;
|
|
|
|
+ return 1;
|
|
|
|
+
|
|
|
|
+no_merge:
|
|
|
|
+ req->cmd_flags |= REQ_NOMERGE;
|
|
|
|
+ if (req == q->last_merge)
|
|
|
|
+ q->last_merge = NULL;
|
|
|
|
+ return 0;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+int ll_back_merge_fn(struct request_queue *q, struct request *req,
|
|
|
|
+ struct bio *bio)
|
|
|
|
+{
|
|
|
|
+ if (blk_rq_sectors(req) + bio_sectors(bio) >
|
|
|
|
+ blk_rq_get_max_sectors(req)) {
|
|
|
|
+ req->cmd_flags |= REQ_NOMERGE;
|
|
|
|
+ if (req == q->last_merge)
|
|
|
|
+ q->last_merge = NULL;
|
|
|
|
+ return 0;
|
|
|
|
+ }
|
|
|
|
+ if (!bio_flagged(req->biotail, BIO_SEG_VALID))
|
|
|
|
+ blk_recount_segments(q, req->biotail);
|
|
|
|
+ if (!bio_flagged(bio, BIO_SEG_VALID))
|
|
|
|
+ blk_recount_segments(q, bio);
|
|
|
|
+
|
|
|
|
+ return ll_new_hw_segment(q, req, bio);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+int ll_front_merge_fn(struct request_queue *q, struct request *req,
|
|
|
|
+ struct bio *bio)
|
|
|
|
+{
|
|
|
|
+ if (blk_rq_sectors(req) + bio_sectors(bio) >
|
|
|
|
+ blk_rq_get_max_sectors(req)) {
|
|
|
|
+ req->cmd_flags |= REQ_NOMERGE;
|
|
|
|
+ if (req == q->last_merge)
|
|
|
|
+ q->last_merge = NULL;
|
|
|
|
+ return 0;
|
|
|
|
+ }
|
|
|
|
+ if (!bio_flagged(bio, BIO_SEG_VALID))
|
|
|
|
+ blk_recount_segments(q, bio);
|
|
|
|
+ if (!bio_flagged(req->bio, BIO_SEG_VALID))
|
|
|
|
+ blk_recount_segments(q, req->bio);
|
|
|
|
+
|
|
|
|
+ return ll_new_hw_segment(q, req, bio);
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
|
|
|
|
+ struct request *next)
|
|
|
|
+{
|
|
|
|
+ int total_phys_segments;
|
|
|
|
+ unsigned int seg_size =
|
|
|
|
+ req->biotail->bi_seg_back_size + next->bio->bi_seg_front_size;
|
|
|
|
+
|
|
|
|
+ /*
|
|
|
|
+ * First check if the either of the requests are re-queued
|
|
|
|
+ * requests. Can't merge them if they are.
|
|
|
|
+ */
|
|
|
|
+ if (req->special || next->special)
|
|
|
|
+ return 0;
|
|
|
|
+
|
|
|
|
+ /*
|
|
|
|
+ * Will it become too large?
|
|
|
|
+ */
|
|
|
|
+ if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
|
|
|
|
+ blk_rq_get_max_sectors(req))
|
|
|
|
+ return 0;
|
|
|
|
+
|
|
|
|
+ total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
|
|
|
|
+ if (blk_phys_contig_segment(q, req->biotail, next->bio)) {
|
|
|
|
+ if (req->nr_phys_segments == 1)
|
|
|
|
+ req->bio->bi_seg_front_size = seg_size;
|
|
|
|
+ if (next->nr_phys_segments == 1)
|
|
|
|
+ next->biotail->bi_seg_back_size = seg_size;
|
|
|
|
+ total_phys_segments--;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (total_phys_segments > queue_max_segments(q))
|
|
|
|
+ return 0;
|
|
|
|
+
|
|
|
|
+ if (blk_integrity_rq(req) && blk_integrity_merge_rq(q, req, next))
|
|
|
|
+ return 0;
|
|
|
|
+
|
|
|
|
+ /* Merge is OK... */
|
|
|
|
+ req->nr_phys_segments = total_phys_segments;
|
|
|
|
+ return 1;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+/**
|
|
|
|
+ * blk_rq_set_mixed_merge - mark a request as mixed merge
|
|
|
|
+ * @rq: request to mark as mixed merge
|
|
|
|
+ *
|
|
|
|
+ * Description:
|
|
|
|
+ * @rq is about to be mixed merged. Make sure the attributes
|
|
|
|
+ * which can be mixed are set in each bio and mark @rq as mixed
|
|
|
|
+ * merged.
|
|
|
|
+ */
|
|
|
|
+void blk_rq_set_mixed_merge(struct request *rq)
|
|
|
|
+{
|
|
|
|
+ unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
|
|
|
|
+ struct bio *bio;
|
|
|
|
+
|
|
|
|
+ if (rq->cmd_flags & REQ_MIXED_MERGE)
|
|
|
|
+ return;
|
|
|
|
+
|
|
|
|
+ /*
|
|
|
|
+ * @rq will no longer represent mixable attributes for all the
|
|
|
|
+ * contained bios. It will just track those of the first one.
|
|
|
|
+ * Distributes the attributs to each bio.
|
|
|
|
+ */
|
|
|
|
+ for (bio = rq->bio; bio; bio = bio->bi_next) {
|
|
|
|
+ WARN_ON_ONCE((bio->bi_rw & REQ_FAILFAST_MASK) &&
|
|
|
|
+ (bio->bi_rw & REQ_FAILFAST_MASK) != ff);
|
|
|
|
+ bio->bi_rw |= ff;
|
|
|
|
+ }
|
|
|
|
+ rq->cmd_flags |= REQ_MIXED_MERGE;
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+static void blk_account_io_merge(struct request *req)
|
|
|
|
+{
|
|
|
|
+ if (blk_do_io_stat(req)) {
|
|
|
|
+ struct hd_struct *part;
|
|
|
|
+ int cpu;
|
|
|
|
+
|
|
|
|
+ cpu = part_stat_lock();
|