| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192 | /* * Functions related to segment and merge handling */#include <linux/kernel.h>#include <linux/module.h>#include <linux/bio.h>#include <linux/blkdev.h>#include <linux/scatterlist.h>#include "blk.h"static unsigned int __blk_recalc_rq_segments(struct request_queue *q,					     struct bio *bio){	struct bio_vec *bv, *bvprv = NULL;	int cluster, i, high, highprv = 1;	unsigned int seg_size, nr_phys_segs;	struct bio *fbio, *bbio;	if (!bio)		return 0;	fbio = bio;	cluster = blk_queue_cluster(q);	seg_size = 0;	nr_phys_segs = 0;	for_each_bio(bio) {		bio_for_each_segment(bv, bio, i) {			/*			 * the trick here is making sure that a high page is			 * never considered part of another segment, since that			 * might change with the bounce page.			 */			high = page_to_pfn(bv->bv_page) > queue_bounce_pfn(q);			if (high || highprv)				goto new_segment;			if (cluster) {				if (seg_size + bv->bv_len				    > queue_max_segment_size(q))					goto new_segment;				if (!BIOVEC_PHYS_MERGEABLE(bvprv, bv))					goto new_segment;				if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bv))					goto new_segment;				seg_size += bv->bv_len;				bvprv = bv;				continue;			}new_segment:			if (nr_phys_segs == 1 && seg_size >			    fbio->bi_seg_front_size)				fbio->bi_seg_front_size = seg_size;			nr_phys_segs++;			bvprv = bv;			seg_size = bv->bv_len;			highprv = high;		}		bbio = bio;	}	if (nr_phys_segs == 1 && seg_size > fbio->bi_seg_front_size)		fbio->bi_seg_front_size = seg_size;	if (seg_size > bbio->bi_seg_back_size)		bbio->bi_seg_back_size = seg_size;	return nr_phys_segs;}void blk_recalc_rq_segments(struct request *rq){	rq->nr_phys_segments = __blk_recalc_rq_segments(rq->q, rq->bio);}void blk_recount_segments(struct request_queue *q, struct bio *bio){	struct bio *nxt = bio->bi_next;	bio->bi_next = NULL;	bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);	bio->bi_next = nxt;	bio->bi_flags |= (1 << BIO_SEG_VALID);}EXPORT_SYMBOL(blk_recount_segments);static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio,				   struct bio *nxt){	if (!blk_queue_cluster(q))		return 0;	if (bio->bi_seg_back_size + nxt->bi_seg_front_size >	    queue_max_segment_size(q))		return 0;	if (!bio_has_data(bio))		return 1;	if (!BIOVEC_PHYS_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt)))		return 0;	/*	 * bio and nxt are contiguous in memory; check if the queue allows	 * these two to be merged into one	 */	if (BIO_SEG_BOUNDARY(q, bio, nxt))		return 1;	return 0;}static void__blk_segment_map_sg(struct request_queue *q, struct bio_vec *bvec,		     struct scatterlist *sglist, struct bio_vec **bvprv,		     struct scatterlist **sg, int *nsegs, int *cluster){	int nbytes = bvec->bv_len;	if (*bvprv && *cluster) {		if ((*sg)->length + nbytes > queue_max_segment_size(q))			goto new_segment;		if (!BIOVEC_PHYS_MERGEABLE(*bvprv, bvec))			goto new_segment;		if (!BIOVEC_SEG_BOUNDARY(q, *bvprv, bvec))			goto new_segment;		(*sg)->length += nbytes;	} else {new_segment:		if (!*sg)			*sg = sglist;		else {			/*			 * If the driver previously mapped a shorter			 * list, we could see a termination bit			 * prematurely unless it fully inits the sg			 * table on each mapping. We KNOW that there			 * must be more entries here or the driver			 * would be buggy, so force clear the			 * termination bit to avoid doing a full			 * sg_init_table() in drivers for each command.			 */			(*sg)->page_link &= ~0x02;			*sg = sg_next(*sg);		}		sg_set_page(*sg, bvec->bv_page, nbytes, bvec->bv_offset);		(*nsegs)++;	}	*bvprv = bvec;}/* * map a request to scatterlist, return number of sg entries setup. Caller * must make sure sg can hold rq->nr_phys_segments entries */int blk_rq_map_sg(struct request_queue *q, struct request *rq,		  struct scatterlist *sglist){	struct bio_vec *bvec, *bvprv;	struct req_iterator iter;	struct scatterlist *sg;	int nsegs, cluster;	nsegs = 0;	cluster = blk_queue_cluster(q);	/*	 * for each bio in rq	 */	bvprv = NULL;	sg = NULL;	rq_for_each_segment(bvec, rq, iter) {		__blk_segment_map_sg(q, bvec, sglist, &bvprv, &sg,				     &nsegs, &cluster);	} /* segments in rq */	if (unlikely(rq->cmd_flags & REQ_COPY_USER) &&	    (blk_rq_bytes(rq) & q->dma_pad_mask)) {		unsigned int pad_len =			(q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;		sg->length += pad_len;		rq->extra_len += pad_len;	}	if (q->dma_drain_size && q->dma_drain_needed(rq)) {		if (rq->cmd_flags & REQ_WRITE)
 |