|
@@ -2424,3 +2424,184 @@ static bool blk_update_bidi_request(struct request *rq, int error,
|
|
|
if (blk_queue_add_random(rq->q))
|
|
|
add_disk_randomness(rq->rq_disk);
|
|
|
|
|
|
+ return false;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_unprep_request - unprepare a request
|
|
|
+ * @req: the request
|
|
|
+ *
|
|
|
+ * This function makes a request ready for complete resubmission (or
|
|
|
+ * completion). It happens only after all error handling is complete,
|
|
|
+ * so represents the appropriate moment to deallocate any resources
|
|
|
+ * that were allocated to the request in the prep_rq_fn. The queue
|
|
|
+ * lock is held when calling this.
|
|
|
+ */
|
|
|
+void blk_unprep_request(struct request *req)
|
|
|
+{
|
|
|
+ struct request_queue *q = req->q;
|
|
|
+
|
|
|
+ req->cmd_flags &= ~REQ_DONTPREP;
|
|
|
+ if (q->unprep_rq_fn)
|
|
|
+ q->unprep_rq_fn(q, req);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(blk_unprep_request);
|
|
|
+
|
|
|
+/*
|
|
|
+ * queue lock must be held
|
|
|
+ */
|
|
|
+static void blk_finish_request(struct request *req, int error)
|
|
|
+{
|
|
|
+ if (blk_rq_tagged(req))
|
|
|
+ blk_queue_end_tag(req->q, req);
|
|
|
+
|
|
|
+ BUG_ON(blk_queued_rq(req));
|
|
|
+
|
|
|
+ if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)
|
|
|
+ laptop_io_completion(&req->q->backing_dev_info);
|
|
|
+
|
|
|
+ blk_delete_timer(req);
|
|
|
+
|
|
|
+ if (req->cmd_flags & REQ_DONTPREP)
|
|
|
+ blk_unprep_request(req);
|
|
|
+
|
|
|
+
|
|
|
+ blk_account_io_done(req);
|
|
|
+
|
|
|
+ if (req->end_io)
|
|
|
+ req->end_io(req, error);
|
|
|
+ else {
|
|
|
+ if (blk_bidi_rq(req))
|
|
|
+ __blk_put_request(req->next_rq->q, req->next_rq);
|
|
|
+
|
|
|
+ __blk_put_request(req->q, req);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_end_bidi_request - Complete a bidi request
|
|
|
+ * @rq: the request to complete
|
|
|
+ * @error: %0 for success, < %0 for error
|
|
|
+ * @nr_bytes: number of bytes to complete @rq
|
|
|
+ * @bidi_bytes: number of bytes to complete @rq->next_rq
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
|
|
|
+ * Drivers that supports bidi can safely call this member for any
|
|
|
+ * type of request, bidi or uni. In the later case @bidi_bytes is
|
|
|
+ * just ignored.
|
|
|
+ *
|
|
|
+ * Return:
|
|
|
+ * %false - we are done with this request
|
|
|
+ * %true - still buffers pending for this request
|
|
|
+ **/
|
|
|
+static bool blk_end_bidi_request(struct request *rq, int error,
|
|
|
+ unsigned int nr_bytes, unsigned int bidi_bytes)
|
|
|
+{
|
|
|
+ struct request_queue *q = rq->q;
|
|
|
+ unsigned long flags;
|
|
|
+
|
|
|
+ if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
|
|
|
+ return true;
|
|
|
+
|
|
|
+ spin_lock_irqsave(q->queue_lock, flags);
|
|
|
+ blk_finish_request(rq, error);
|
|
|
+ spin_unlock_irqrestore(q->queue_lock, flags);
|
|
|
+
|
|
|
+ return false;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * __blk_end_bidi_request - Complete a bidi request with queue lock held
|
|
|
+ * @rq: the request to complete
|
|
|
+ * @error: %0 for success, < %0 for error
|
|
|
+ * @nr_bytes: number of bytes to complete @rq
|
|
|
+ * @bidi_bytes: number of bytes to complete @rq->next_rq
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Identical to blk_end_bidi_request() except that queue lock is
|
|
|
+ * assumed to be locked on entry and remains so on return.
|
|
|
+ *
|
|
|
+ * Return:
|
|
|
+ * %false - we are done with this request
|
|
|
+ * %true - still buffers pending for this request
|
|
|
+ **/
|
|
|
+bool __blk_end_bidi_request(struct request *rq, int error,
|
|
|
+ unsigned int nr_bytes, unsigned int bidi_bytes)
|
|
|
+{
|
|
|
+ if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
|
|
|
+ return true;
|
|
|
+
|
|
|
+ blk_finish_request(rq, error);
|
|
|
+
|
|
|
+ return false;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_end_request - Helper function for drivers to complete the request.
|
|
|
+ * @rq: the request being processed
|
|
|
+ * @error: %0 for success, < %0 for error
|
|
|
+ * @nr_bytes: number of bytes to complete
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Ends I/O on a number of bytes attached to @rq.
|
|
|
+ * If @rq has leftover, sets it up for the next range of segments.
|
|
|
+ *
|
|
|
+ * Return:
|
|
|
+ * %false - we are done with this request
|
|
|
+ * %true - still buffers pending for this request
|
|
|
+ **/
|
|
|
+bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
|
|
|
+{
|
|
|
+ return blk_end_bidi_request(rq, error, nr_bytes, 0);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_end_request);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_end_request_all - Helper function for drives to finish the request.
|
|
|
+ * @rq: the request to finish
|
|
|
+ * @error: %0 for success, < %0 for error
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Completely finish @rq.
|
|
|
+ */
|
|
|
+void blk_end_request_all(struct request *rq, int error)
|
|
|
+{
|
|
|
+ bool pending;
|
|
|
+ unsigned int bidi_bytes = 0;
|
|
|
+
|
|
|
+ if (unlikely(blk_bidi_rq(rq)))
|
|
|
+ bidi_bytes = blk_rq_bytes(rq->next_rq);
|
|
|
+
|
|
|
+ pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
|
|
|
+ BUG_ON(pending);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_end_request_all);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_end_request_cur - Helper function to finish the current request chunk.
|
|
|
+ * @rq: the request to finish the current chunk for
|
|
|
+ * @error: %0 for success, < %0 for error
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Complete the current consecutively mapped chunk from @rq.
|
|
|
+ *
|
|
|
+ * Return:
|
|
|
+ * %false - we are done with this request
|
|
|
+ * %true - still buffers pending for this request
|
|
|
+ */
|
|
|
+bool blk_end_request_cur(struct request *rq, int error)
|
|
|
+{
|
|
|
+ return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_end_request_cur);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_end_request_err - Finish a request till the next failure boundary.
|
|
|
+ * @rq: the request to finish till the next failure boundary for
|
|
|
+ * @error: must be negative errno
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Complete @rq till the next failure boundary.
|
|
|
+ *
|
|
|
+ * Return:
|