|  | @@ -1318,3 +1318,166 @@ EXPORT_SYMBOL(blk_put_request);
 | 
	
		
			
				|  |  |   * a block driver.  The driver needs to take care of freeing the payload
 | 
	
		
			
				|  |  |   * itself.
 | 
	
		
			
				|  |  |   *
 | 
	
		
			
				|  |  | + * Note that this is a quite horrible hack and nothing but handling of
 | 
	
		
			
				|  |  | + * discard requests should ever use it.
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +void blk_add_request_payload(struct request *rq, struct page *page,
 | 
	
		
			
				|  |  | +		unsigned int len)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	struct bio *bio = rq->bio;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	bio->bi_io_vec->bv_page = page;
 | 
	
		
			
				|  |  | +	bio->bi_io_vec->bv_offset = 0;
 | 
	
		
			
				|  |  | +	bio->bi_io_vec->bv_len = len;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	bio->bi_size = len;
 | 
	
		
			
				|  |  | +	bio->bi_vcnt = 1;
 | 
	
		
			
				|  |  | +	bio->bi_phys_segments = 1;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	rq->__data_len = rq->resid_len = len;
 | 
	
		
			
				|  |  | +	rq->nr_phys_segments = 1;
 | 
	
		
			
				|  |  | +	rq->buffer = bio_data(bio);
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL_GPL(blk_add_request_payload);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
 | 
	
		
			
				|  |  | +				   struct bio *bio)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if (!ll_back_merge_fn(q, req, bio))
 | 
	
		
			
				|  |  | +		return false;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	trace_block_bio_backmerge(q, bio);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
 | 
	
		
			
				|  |  | +		blk_rq_set_mixed_merge(req);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	req->biotail->bi_next = bio;
 | 
	
		
			
				|  |  | +	req->biotail = bio;
 | 
	
		
			
				|  |  | +	req->__data_len += bio->bi_size;
 | 
	
		
			
				|  |  | +	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	drive_stat_acct(req, 0);
 | 
	
		
			
				|  |  | +	return true;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static bool bio_attempt_front_merge(struct request_queue *q,
 | 
	
		
			
				|  |  | +				    struct request *req, struct bio *bio)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if (!ll_front_merge_fn(q, req, bio))
 | 
	
		
			
				|  |  | +		return false;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	trace_block_bio_frontmerge(q, bio);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
 | 
	
		
			
				|  |  | +		blk_rq_set_mixed_merge(req);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	bio->bi_next = req->bio;
 | 
	
		
			
				|  |  | +	req->bio = bio;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * may not be valid. if the low level driver said
 | 
	
		
			
				|  |  | +	 * it didn't need a bounce buffer then it better
 | 
	
		
			
				|  |  | +	 * not touch req->buffer either...
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	req->buffer = bio_data(bio);
 | 
	
		
			
				|  |  | +	req->__sector = bio->bi_sector;
 | 
	
		
			
				|  |  | +	req->__data_len += bio->bi_size;
 | 
	
		
			
				|  |  | +	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	drive_stat_acct(req, 0);
 | 
	
		
			
				|  |  | +	return true;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/**
 | 
	
		
			
				|  |  | + * attempt_plug_merge - try to merge with %current's plugged list
 | 
	
		
			
				|  |  | + * @q: request_queue new bio is being queued at
 | 
	
		
			
				|  |  | + * @bio: new bio being queued
 | 
	
		
			
				|  |  | + * @request_count: out parameter for number of traversed plugged requests
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Determine whether @bio being queued on @q can be merged with a request
 | 
	
		
			
				|  |  | + * on %current's plugged list.  Returns %true if merge was successful,
 | 
	
		
			
				|  |  | + * otherwise %false.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Plugging coalesces IOs from the same issuer for the same purpose without
 | 
	
		
			
				|  |  | + * going through @q->queue_lock.  As such it's more of an issuing mechanism
 | 
	
		
			
				|  |  | + * than scheduling, and the request, while may have elvpriv data, is not
 | 
	
		
			
				|  |  | + * added on the elevator at this point.  In addition, we don't have
 | 
	
		
			
				|  |  | + * reliable access to the elevator outside queue lock.  Only check basic
 | 
	
		
			
				|  |  | + * merging parameters without querying the elevator.
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +static bool attempt_plug_merge(struct request_queue *q, struct bio *bio,
 | 
	
		
			
				|  |  | +			       unsigned int *request_count)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	struct blk_plug *plug;
 | 
	
		
			
				|  |  | +	struct request *rq;
 | 
	
		
			
				|  |  | +	bool ret = false;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	plug = current->plug;
 | 
	
		
			
				|  |  | +	if (!plug)
 | 
	
		
			
				|  |  | +		goto out;
 | 
	
		
			
				|  |  | +	*request_count = 0;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	list_for_each_entry_reverse(rq, &plug->list, queuelist) {
 | 
	
		
			
				|  |  | +		int el_ret;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		if (rq->q == q)
 | 
	
		
			
				|  |  | +			(*request_count)++;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
 | 
	
		
			
				|  |  | +			continue;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		el_ret = blk_try_merge(rq, bio);
 | 
	
		
			
				|  |  | +		if (el_ret == ELEVATOR_BACK_MERGE) {
 | 
	
		
			
				|  |  | +			ret = bio_attempt_back_merge(q, rq, bio);
 | 
	
		
			
				|  |  | +			if (ret)
 | 
	
		
			
				|  |  | +				break;
 | 
	
		
			
				|  |  | +		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
 | 
	
		
			
				|  |  | +			ret = bio_attempt_front_merge(q, rq, bio);
 | 
	
		
			
				|  |  | +			if (ret)
 | 
	
		
			
				|  |  | +				break;
 | 
	
		
			
				|  |  | +		}
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +out:
 | 
	
		
			
				|  |  | +	return ret;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +void init_request_from_bio(struct request *req, struct bio *bio)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	req->cmd_type = REQ_TYPE_FS;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;
 | 
	
		
			
				|  |  | +	if (bio->bi_rw & REQ_RAHEAD)
 | 
	
		
			
				|  |  | +		req->cmd_flags |= REQ_FAILFAST_MASK;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	req->errors = 0;
 | 
	
		
			
				|  |  | +	req->__sector = bio->bi_sector;
 | 
	
		
			
				|  |  | +	req->ioprio = bio_prio(bio);
 | 
	
		
			
				|  |  | +	blk_rq_bio_prep(req->q, req, bio);
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +void blk_queue_bio(struct request_queue *q, struct bio *bio)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	const bool sync = !!(bio->bi_rw & REQ_SYNC);
 | 
	
		
			
				|  |  | +	struct blk_plug *plug;
 | 
	
		
			
				|  |  | +	int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT;
 | 
	
		
			
				|  |  | +	struct request *req;
 | 
	
		
			
				|  |  | +	unsigned int request_count = 0;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * low level driver can indicate that it wants pages above a
 | 
	
		
			
				|  |  | +	 * certain limit bounced to low memory (ie for highmem, or even
 | 
	
		
			
				|  |  | +	 * ISA dma in theory)
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	blk_queue_bounce(q, &bio);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
 | 
	
		
			
				|  |  | +		spin_lock_irq(q->queue_lock);
 | 
	
		
			
				|  |  | +		where = ELEVATOR_INSERT_FLUSH;
 | 
	
		
			
				|  |  | +		goto get_rq;
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 |