|
@@ -268,3 +268,188 @@ static bool blk_kick_flush(struct request_queue *q)
|
|
|
q->flush_rq.end_io = flush_end_io;
|
|
|
|
|
|
q->flush_pending_idx ^= 1;
|
|
|
+ list_add_tail(&q->flush_rq.queuelist, &q->queue_head);
|
|
|
+ return true;
|
|
|
+}
|
|
|
+
|
|
|
+static void flush_data_end_io(struct request *rq, int error)
|
|
|
+{
|
|
|
+ struct request_queue *q = rq->q;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * After populating an empty queue, kick it to avoid stall. Read
|
|
|
+ * the comment in flush_end_io().
|
|
|
+ */
|
|
|
+ if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error))
|
|
|
+ blk_run_queue_async(q);
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_insert_flush - insert a new FLUSH/FUA request
|
|
|
+ * @rq: request to insert
|
|
|
+ *
|
|
|
+ * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions.
|
|
|
+ * @rq is being submitted. Analyze what needs to be done and put it on the
|
|
|
+ * right queue.
|
|
|
+ *
|
|
|
+ * CONTEXT:
|
|
|
+ * spin_lock_irq(q->queue_lock)
|
|
|
+ */
|
|
|
+void blk_insert_flush(struct request *rq)
|
|
|
+{
|
|
|
+ struct request_queue *q = rq->q;
|
|
|
+ unsigned int fflags = q->flush_flags; /* may change, cache */
|
|
|
+ unsigned int policy = blk_flush_policy(fflags, rq);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * @policy now records what operations need to be done. Adjust
|
|
|
+ * REQ_FLUSH and FUA for the driver.
|
|
|
+ */
|
|
|
+ rq->cmd_flags &= ~REQ_FLUSH;
|
|
|
+ if (!(fflags & REQ_FUA))
|
|
|
+ rq->cmd_flags &= ~REQ_FUA;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * An empty flush handed down from a stacking driver may
|
|
|
+ * translate into nothing if the underlying device does not
|
|
|
+ * advertise a write-back cache. In this case, simply
|
|
|
+ * complete the request.
|
|
|
+ */
|
|
|
+ if (!policy) {
|
|
|
+ __blk_end_bidi_request(rq, 0, 0, 0);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */
|
|
|
+
|
|
|
+ /*
|
|
|
+ * If there's data but flush is not necessary, the request can be
|
|
|
+ * processed directly without going through flush machinery. Queue
|
|
|
+ * for normal execution.
|
|
|
+ */
|
|
|
+ if ((policy & REQ_FSEQ_DATA) &&
|
|
|
+ !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
|
|
|
+ list_add_tail(&rq->queuelist, &q->queue_head);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * @rq should go through flush machinery. Mark it part of flush
|
|
|
+ * sequence and submit for further processing.
|
|
|
+ */
|
|
|
+ memset(&rq->flush, 0, sizeof(rq->flush));
|
|
|
+ INIT_LIST_HEAD(&rq->flush.list);
|
|
|
+ rq->cmd_flags |= REQ_FLUSH_SEQ;
|
|
|
+ rq->flush.saved_end_io = rq->end_io; /* Usually NULL */
|
|
|
+ rq->end_io = flush_data_end_io;
|
|
|
+
|
|
|
+ blk_flush_complete_seq(rq, REQ_FSEQ_ACTIONS & ~policy, 0);
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_abort_flushes - @q is being aborted, abort flush requests
|
|
|
+ * @q: request_queue being aborted
|
|
|
+ *
|
|
|
+ * To be called from elv_abort_queue(). @q is being aborted. Prepare all
|
|
|
+ * FLUSH/FUA requests for abortion.
|
|
|
+ *
|
|
|
+ * CONTEXT:
|
|
|
+ * spin_lock_irq(q->queue_lock)
|
|
|
+ */
|
|
|
+void blk_abort_flushes(struct request_queue *q)
|
|
|
+{
|
|
|
+ struct request *rq, *n;
|
|
|
+ int i;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Requests in flight for data are already owned by the dispatch
|
|
|
+ * queue or the device driver. Just restore for normal completion.
|
|
|
+ */
|
|
|
+ list_for_each_entry_safe(rq, n, &q->flush_data_in_flight, flush.list) {
|
|
|
+ list_del_init(&rq->flush.list);
|
|
|
+ blk_flush_restore_request(rq);
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * We need to give away requests on flush queues. Restore for
|
|
|
+ * normal completion and put them on the dispatch queue.
|
|
|
+ */
|
|
|
+ for (i = 0; i < ARRAY_SIZE(q->flush_queue); i++) {
|
|
|
+ list_for_each_entry_safe(rq, n, &q->flush_queue[i],
|
|
|
+ flush.list) {
|
|
|
+ list_del_init(&rq->flush.list);
|
|
|
+ blk_flush_restore_request(rq);
|
|
|
+ list_add_tail(&rq->queuelist, &q->queue_head);
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static void bio_end_flush(struct bio *bio, int err)
|
|
|
+{
|
|
|
+ if (err)
|
|
|
+ clear_bit(BIO_UPTODATE, &bio->bi_flags);
|
|
|
+ if (bio->bi_private)
|
|
|
+ complete(bio->bi_private);
|
|
|
+ bio_put(bio);
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blkdev_issue_flush - queue a flush
|
|
|
+ * @bdev: blockdev to issue flush for
|
|
|
+ * @gfp_mask: memory allocation flags (for bio_alloc)
|
|
|
+ * @error_sector: error sector
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Issue a flush for the block device in question. Caller can supply
|
|
|
+ * room for storing the error offset in case of a flush error, if they
|
|
|
+ * wish to. If WAIT flag is not passed then caller may check only what
|
|
|
+ * request was pushed in some internal queue for later handling.
|
|
|
+ */
|
|
|
+int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
|
|
|
+ sector_t *error_sector)
|
|
|
+{
|
|
|
+ DECLARE_COMPLETION_ONSTACK(wait);
|
|
|
+ struct request_queue *q;
|
|
|
+ struct bio *bio;
|
|
|
+ int ret = 0;
|
|
|
+
|
|
|
+ if (bdev->bd_disk == NULL)
|
|
|
+ return -ENXIO;
|
|
|
+
|
|
|
+ q = bdev_get_queue(bdev);
|
|
|
+ if (!q)
|
|
|
+ return -ENXIO;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * some block devices may not have their queue correctly set up here
|
|
|
+ * (e.g. loop device without a backing file) and so issuing a flush
|
|
|
+ * here will panic. Ensure there is a request function before issuing
|
|
|
+ * the flush.
|
|
|
+ */
|
|
|
+ if (!q->make_request_fn)
|
|
|
+ return -ENXIO;
|
|
|
+
|
|
|
+ bio = bio_alloc(gfp_mask, 0);
|
|
|
+ bio->bi_end_io = bio_end_flush;
|
|
|
+ bio->bi_bdev = bdev;
|
|
|
+ bio->bi_private = &wait;
|
|
|
+
|
|
|
+ bio_get(bio);
|
|
|
+ submit_bio(WRITE_FLUSH, bio);
|
|
|
+ wait_for_completion(&wait);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * The driver must store the error location in ->bi_sector, if
|
|
|
+ * it supports it. For non-stacked drivers, this should be
|
|
|
+ * copied from blk_rq_pos(rq).
|
|
|
+ */
|
|
|
+ if (error_sector)
|
|
|
+ *error_sector = bio->bi_sector;
|
|
|
+
|
|
|
+ if (!bio_flagged(bio, BIO_UPTODATE))
|
|
|
+ ret = -EIO;
|
|
|
+
|
|
|
+ bio_put(bio);
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blkdev_issue_flush);
|