| 
					
				 | 
			
			
				@@ -178,3 +178,93 @@ static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 		 * flush data request completion path.  Restore @rq for 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 		 * normal completion and end it. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 		 */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		BUG_ON(!list_empty(&rq->queuelist)); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		list_del_init(&rq->flush.list); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		blk_flush_restore_request(rq); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		__blk_end_request_all(rq, error); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		break; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	default: 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		BUG(); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return blk_kick_flush(q) | queued; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static void flush_end_io(struct request *flush_rq, int error) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct request_queue *q = flush_rq->q; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct list_head *running = &q->flush_queue[q->flush_running_idx]; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	bool queued = false; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct request *rq, *n; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	BUG_ON(q->flush_pending_idx == q->flush_running_idx); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* account completion of the flush request */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	q->flush_running_idx ^= 1; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	elv_completed_request(q, flush_rq); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* and push the waiting requests to the next stage */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	list_for_each_entry_safe(rq, n, running, flush.list) { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		unsigned int seq = blk_flush_cur_seq(rq); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		queued |= blk_flush_complete_seq(rq, seq, error); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * Kick the queue to avoid stall for two cases: 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * 1. Moving a request silently to empty queue_head may stall the 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * queue. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * 2. When flush request is running in non-queueable queue, the 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * queue is hold. Restart the queue after flush request is finished 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * to avoid stall. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * This function is called from request completion path and calling 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * directly into request_fn may confuse the driver.  Always use 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * kblockd. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (queued || q->flush_queue_delayed) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		blk_run_queue_async(q); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	q->flush_queue_delayed = 0; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+/** 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * blk_kick_flush - consider issuing flush request 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * @q: request_queue being kicked 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * Flush related states of @q have changed, consider issuing flush request. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * Please read the comment at the top of this file for more info. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * CONTEXT: 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * spin_lock_irq(q->queue_lock) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * RETURNS: 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * %true if flush was issued, %false otherwise. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static bool blk_kick_flush(struct request_queue *q) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct list_head *pending = &q->flush_queue[q->flush_pending_idx]; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct request *first_rq = 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		list_first_entry(pending, struct request, flush.list); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* C1 described at the top of this file */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (q->flush_pending_idx != q->flush_running_idx || list_empty(pending)) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return false; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* C2 and C3 */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (!list_empty(&q->flush_data_in_flight) && 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	    time_before(jiffies, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+			q->flush_pending_since + FLUSH_PENDING_TIMEOUT)) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return false; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * Issue flush and toggle pending_idx.  This makes pending_idx 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * different from running_idx, which means flush is in flight. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	blk_rq_init(q, &q->flush_rq); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	q->flush_rq.cmd_type = REQ_TYPE_FS; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	q->flush_rq.cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	q->flush_rq.rq_disk = first_rq->rq_disk; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	q->flush_rq.end_io = flush_end_io; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	q->flush_pending_idx ^= 1; 
			 |