|
@@ -435,3 +435,140 @@ static void __blk_drain_queue(struct request_queue *q, bool drain_all)
|
|
|
}
|
|
|
|
|
|
if (!drain)
|
|
|
+ break;
|
|
|
+
|
|
|
+ spin_unlock_irq(q->queue_lock);
|
|
|
+
|
|
|
+ msleep(10);
|
|
|
+
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * With queue marked dead, any woken up waiter will fail the
|
|
|
+ * allocation path, so the wakeup chaining is lost and we're
|
|
|
+ * left with hung waiters. We need to wake up those waiters.
|
|
|
+ */
|
|
|
+ if (q->request_fn) {
|
|
|
+ struct request_list *rl;
|
|
|
+
|
|
|
+ blk_queue_for_each_rl(rl, q)
|
|
|
+ for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
|
|
|
+ wake_up_all(&rl->wait[i]);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_queue_bypass_start - enter queue bypass mode
|
|
|
+ * @q: queue of interest
|
|
|
+ *
|
|
|
+ * In bypass mode, only the dispatch FIFO queue of @q is used. This
|
|
|
+ * function makes @q enter bypass mode and drains all requests which were
|
|
|
+ * throttled or issued before. On return, it's guaranteed that no request
|
|
|
+ * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
|
|
|
+ * inside queue or RCU read lock.
|
|
|
+ */
|
|
|
+void blk_queue_bypass_start(struct request_queue *q)
|
|
|
+{
|
|
|
+ bool drain;
|
|
|
+
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ drain = !q->bypass_depth++;
|
|
|
+ queue_flag_set(QUEUE_FLAG_BYPASS, q);
|
|
|
+ spin_unlock_irq(q->queue_lock);
|
|
|
+
|
|
|
+ if (drain) {
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ __blk_drain_queue(q, false);
|
|
|
+ spin_unlock_irq(q->queue_lock);
|
|
|
+
|
|
|
+ /* ensure blk_queue_bypass() is %true inside RCU read lock */
|
|
|
+ synchronize_rcu();
|
|
|
+ }
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_queue_bypass_end - leave queue bypass mode
|
|
|
+ * @q: queue of interest
|
|
|
+ *
|
|
|
+ * Leave bypass mode and restore the normal queueing behavior.
|
|
|
+ */
|
|
|
+void blk_queue_bypass_end(struct request_queue *q)
|
|
|
+{
|
|
|
+ spin_lock_irq(q->queue_lock);
|
|
|
+ if (!--q->bypass_depth)
|
|
|
+ queue_flag_clear(QUEUE_FLAG_BYPASS, q);
|
|
|
+ WARN_ON_ONCE(q->bypass_depth < 0);
|
|
|
+ spin_unlock_irq(q->queue_lock);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_cleanup_queue - shutdown a request queue
|
|
|
+ * @q: request queue to shutdown
|
|
|
+ *
|
|
|
+ * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
|
|
|
+ * put it. All future requests will be failed immediately with -ENODEV.
|
|
|
+ */
|
|
|
+void blk_cleanup_queue(struct request_queue *q)
|
|
|
+{
|
|
|
+ spinlock_t *lock = q->queue_lock;
|
|
|
+
|
|
|
+ /* mark @q DYING, no new request or merges will be allowed afterwards */
|
|
|
+ mutex_lock(&q->sysfs_lock);
|
|
|
+ queue_flag_set_unlocked(QUEUE_FLAG_DYING, q);
|
|
|
+ spin_lock_irq(lock);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * A dying queue is permanently in bypass mode till released. Note
|
|
|
+ * that, unlike blk_queue_bypass_start(), we aren't performing
|
|
|
+ * synchronize_rcu() after entering bypass mode to avoid the delay
|
|
|
+ * as some drivers create and destroy a lot of queues while
|
|
|
+ * probing. This is still safe because blk_release_queue() will be
|
|
|
+ * called only after the queue refcnt drops to zero and nothing,
|
|
|
+ * RCU or not, would be traversing the queue by then.
|
|
|
+ */
|
|
|
+ q->bypass_depth++;
|
|
|
+ queue_flag_set(QUEUE_FLAG_BYPASS, q);
|
|
|
+
|
|
|
+ queue_flag_set(QUEUE_FLAG_NOMERGES, q);
|
|
|
+ queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
|
|
|
+ queue_flag_set(QUEUE_FLAG_DYING, q);
|
|
|
+ spin_unlock_irq(lock);
|
|
|
+ mutex_unlock(&q->sysfs_lock);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Drain all requests queued before DYING marking. Set DEAD flag to
|
|
|
+ * prevent that q->request_fn() gets invoked after draining finished.
|
|
|
+ */
|
|
|
+ spin_lock_irq(lock);
|
|
|
+ __blk_drain_queue(q, true);
|
|
|
+ queue_flag_set(QUEUE_FLAG_DEAD, q);
|
|
|
+ spin_unlock_irq(lock);
|
|
|
+
|
|
|
+ /* @q won't process any more request, flush async actions */
|
|
|
+ del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
|
|
|
+ blk_sync_queue(q);
|
|
|
+
|
|
|
+ spin_lock_irq(lock);
|
|
|
+ if (q->queue_lock != &q->__queue_lock)
|
|
|
+ q->queue_lock = &q->__queue_lock;
|
|
|
+ spin_unlock_irq(lock);
|
|
|
+
|
|
|
+ /* @q is and will stay empty, shutdown and put */
|
|
|
+ blk_put_queue(q);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_cleanup_queue);
|
|
|
+
|
|
|
+int blk_init_rl(struct request_list *rl, struct request_queue *q,
|
|
|
+ gfp_t gfp_mask)
|
|
|
+{
|
|
|
+ if (unlikely(rl->rq_pool))
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ rl->q = q;
|
|
|
+ rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
|
|
|
+ rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
|
|
|
+ init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
|
|
|
+ init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
|