|  | @@ -1950,3 +1950,158 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
 | 
	
		
			
				|  |  |  	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
 | 
	
		
			
				|  |  |  		return -EIO;
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | +	spin_lock_irqsave(q->queue_lock, flags);
 | 
	
		
			
				|  |  | +	if (unlikely(blk_queue_dying(q))) {
 | 
	
		
			
				|  |  | +		spin_unlock_irqrestore(q->queue_lock, flags);
 | 
	
		
			
				|  |  | +		return -ENODEV;
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Submitting request must be dequeued before calling this function
 | 
	
		
			
				|  |  | +	 * because it will be linked to another request_queue
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	BUG_ON(blk_queued_rq(rq));
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA))
 | 
	
		
			
				|  |  | +		where = ELEVATOR_INSERT_FLUSH;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	add_acct_request(q, rq, where);
 | 
	
		
			
				|  |  | +	if (where == ELEVATOR_INSERT_FLUSH)
 | 
	
		
			
				|  |  | +		__blk_run_queue(q);
 | 
	
		
			
				|  |  | +	spin_unlock_irqrestore(q->queue_lock, flags);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	return 0;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/**
 | 
	
		
			
				|  |  | + * blk_rq_err_bytes - determine number of bytes till the next failure boundary
 | 
	
		
			
				|  |  | + * @rq: request to examine
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Description:
 | 
	
		
			
				|  |  | + *     A request could be merge of IOs which require different failure
 | 
	
		
			
				|  |  | + *     handling.  This function determines the number of bytes which
 | 
	
		
			
				|  |  | + *     can be failed from the beginning of the request without
 | 
	
		
			
				|  |  | + *     crossing into area which need to be retried further.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Return:
 | 
	
		
			
				|  |  | + *     The number of bytes to fail.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Context:
 | 
	
		
			
				|  |  | + *     queue_lock must be held.
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +unsigned int blk_rq_err_bytes(const struct request *rq)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
 | 
	
		
			
				|  |  | +	unsigned int bytes = 0;
 | 
	
		
			
				|  |  | +	struct bio *bio;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	if (!(rq->cmd_flags & REQ_MIXED_MERGE))
 | 
	
		
			
				|  |  | +		return blk_rq_bytes(rq);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Currently the only 'mixing' which can happen is between
 | 
	
		
			
				|  |  | +	 * different fastfail types.  We can safely fail portions
 | 
	
		
			
				|  |  | +	 * which have all the failfast bits that the first one has -
 | 
	
		
			
				|  |  | +	 * the ones which are at least as eager to fail as the first
 | 
	
		
			
				|  |  | +	 * one.
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	for (bio = rq->bio; bio; bio = bio->bi_next) {
 | 
	
		
			
				|  |  | +		if ((bio->bi_rw & ff) != ff)
 | 
	
		
			
				|  |  | +			break;
 | 
	
		
			
				|  |  | +		bytes += bio->bi_size;
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	/* this could lead to infinite loop */
 | 
	
		
			
				|  |  | +	BUG_ON(blk_rq_bytes(rq) && !bytes);
 | 
	
		
			
				|  |  | +	return bytes;
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static void blk_account_io_completion(struct request *req, unsigned int bytes)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	if (blk_do_io_stat(req)) {
 | 
	
		
			
				|  |  | +		const int rw = rq_data_dir(req);
 | 
	
		
			
				|  |  | +		struct hd_struct *part;
 | 
	
		
			
				|  |  | +		int cpu;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		cpu = part_stat_lock();
 | 
	
		
			
				|  |  | +		part = req->part;
 | 
	
		
			
				|  |  | +		part_stat_add(cpu, part, sectors[rw], bytes >> 9);
 | 
	
		
			
				|  |  | +		part_stat_unlock();
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +static void blk_account_io_done(struct request *req)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	/*
 | 
	
		
			
				|  |  | +	 * Account IO completion.  flush_rq isn't accounted as a
 | 
	
		
			
				|  |  | +	 * normal IO on queueing nor completion.  Accounting the
 | 
	
		
			
				|  |  | +	 * containing request is enough.
 | 
	
		
			
				|  |  | +	 */
 | 
	
		
			
				|  |  | +	if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {
 | 
	
		
			
				|  |  | +		unsigned long duration = jiffies - req->start_time;
 | 
	
		
			
				|  |  | +		const int rw = rq_data_dir(req);
 | 
	
		
			
				|  |  | +		struct hd_struct *part;
 | 
	
		
			
				|  |  | +		int cpu;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		cpu = part_stat_lock();
 | 
	
		
			
				|  |  | +		part = req->part;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		part_stat_inc(cpu, part, ios[rw]);
 | 
	
		
			
				|  |  | +		part_stat_add(cpu, part, ticks[rw], duration);
 | 
	
		
			
				|  |  | +		part_round_stats(cpu, part);
 | 
	
		
			
				|  |  | +		part_dec_in_flight(part, rw);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		hd_struct_put(part);
 | 
	
		
			
				|  |  | +		part_stat_unlock();
 | 
	
		
			
				|  |  | +	}
 | 
	
		
			
				|  |  | +}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/**
 | 
	
		
			
				|  |  | + * blk_peek_request - peek at the top of a request queue
 | 
	
		
			
				|  |  | + * @q: request queue to peek at
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Description:
 | 
	
		
			
				|  |  | + *     Return the request at the top of @q.  The returned request
 | 
	
		
			
				|  |  | + *     should be started using blk_start_request() before LLD starts
 | 
	
		
			
				|  |  | + *     processing it.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Return:
 | 
	
		
			
				|  |  | + *     Pointer to the request at the top of @q if available.  Null
 | 
	
		
			
				|  |  | + *     otherwise.
 | 
	
		
			
				|  |  | + *
 | 
	
		
			
				|  |  | + * Context:
 | 
	
		
			
				|  |  | + *     queue_lock must be held.
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +struct request *blk_peek_request(struct request_queue *q)
 | 
	
		
			
				|  |  | +{
 | 
	
		
			
				|  |  | +	struct request *rq;
 | 
	
		
			
				|  |  | +	int ret;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +	while ((rq = __elv_next_request(q)) != NULL) {
 | 
	
		
			
				|  |  | +		if (!(rq->cmd_flags & REQ_STARTED)) {
 | 
	
		
			
				|  |  | +			/*
 | 
	
		
			
				|  |  | +			 * This is the first time the device driver
 | 
	
		
			
				|  |  | +			 * sees this request (possibly after
 | 
	
		
			
				|  |  | +			 * requeueing).  Notify IO scheduler.
 | 
	
		
			
				|  |  | +			 */
 | 
	
		
			
				|  |  | +			if (rq->cmd_flags & REQ_SORTED)
 | 
	
		
			
				|  |  | +				elv_activate_rq(q, rq);
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +			/*
 | 
	
		
			
				|  |  | +			 * just mark as started even if we don't start
 | 
	
		
			
				|  |  | +			 * it, a request that has been delayed should
 | 
	
		
			
				|  |  | +			 * not be passed by new incoming requests
 | 
	
		
			
				|  |  | +			 */
 | 
	
		
			
				|  |  | +			rq->cmd_flags |= REQ_STARTED;
 | 
	
		
			
				|  |  | +			trace_block_rq_issue(q, rq);
 | 
	
		
			
				|  |  | +		}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		if (!q->boundary_rq || q->boundary_rq == rq) {
 | 
	
		
			
				|  |  | +			q->end_sector = rq_end_sector(rq);
 | 
	
		
			
				|  |  | +			q->boundary_rq = NULL;
 | 
	
		
			
				|  |  | +		}
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +		if (rq->cmd_flags & REQ_DONTPREP)
 | 
	
		
			
				|  |  | +			break;
 |