|
@@ -241,3 +241,157 @@ int blk_queue_resize_tags(struct request_queue *q, int new_depth)
|
|
|
* one, so error out if this is the case
|
|
|
*/
|
|
|
if (atomic_read(&bqt->refcnt) != 1)
|
|
|
+ return -EBUSY;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * save the old state info, so we can copy it back
|
|
|
+ */
|
|
|
+ tag_index = bqt->tag_index;
|
|
|
+ tag_map = bqt->tag_map;
|
|
|
+ max_depth = bqt->real_max_depth;
|
|
|
+
|
|
|
+ if (init_tag_map(q, bqt, new_depth))
|
|
|
+ return -ENOMEM;
|
|
|
+
|
|
|
+ memcpy(bqt->tag_index, tag_index, max_depth * sizeof(struct request *));
|
|
|
+ nr_ulongs = ALIGN(max_depth, BITS_PER_LONG) / BITS_PER_LONG;
|
|
|
+ memcpy(bqt->tag_map, tag_map, nr_ulongs * sizeof(unsigned long));
|
|
|
+
|
|
|
+ kfree(tag_index);
|
|
|
+ kfree(tag_map);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_queue_resize_tags);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_queue_end_tag - end tag operations for a request
|
|
|
+ * @q: the request queue for the device
|
|
|
+ * @rq: the request that has completed
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Typically called when end_that_request_first() returns %0, meaning
|
|
|
+ * all transfers have been done for a request. It's important to call
|
|
|
+ * this function before end_that_request_last(), as that will put the
|
|
|
+ * request back on the free list thus corrupting the internal tag list.
|
|
|
+ *
|
|
|
+ * Notes:
|
|
|
+ * queue lock must be held.
|
|
|
+ **/
|
|
|
+void blk_queue_end_tag(struct request_queue *q, struct request *rq)
|
|
|
+{
|
|
|
+ struct blk_queue_tag *bqt = q->queue_tags;
|
|
|
+ unsigned tag = rq->tag; /* negative tags invalid */
|
|
|
+
|
|
|
+ BUG_ON(tag >= bqt->real_max_depth);
|
|
|
+
|
|
|
+ list_del_init(&rq->queuelist);
|
|
|
+ rq->cmd_flags &= ~REQ_QUEUED;
|
|
|
+ rq->tag = -1;
|
|
|
+
|
|
|
+ if (unlikely(bqt->tag_index[tag] == NULL))
|
|
|
+ printk(KERN_ERR "%s: tag %d is missing\n",
|
|
|
+ __func__, tag);
|
|
|
+
|
|
|
+ bqt->tag_index[tag] = NULL;
|
|
|
+
|
|
|
+ if (unlikely(!test_bit(tag, bqt->tag_map))) {
|
|
|
+ printk(KERN_ERR "%s: attempt to clear non-busy tag (%d)\n",
|
|
|
+ __func__, tag);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ /*
|
|
|
+ * The tag_map bit acts as a lock for tag_index[bit], so we need
|
|
|
+ * unlock memory barrier semantics.
|
|
|
+ */
|
|
|
+ clear_bit_unlock(tag, bqt->tag_map);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_queue_end_tag);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_queue_start_tag - find a free tag and assign it
|
|
|
+ * @q: the request queue for the device
|
|
|
+ * @rq: the block request that needs tagging
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * This can either be used as a stand-alone helper, or possibly be
|
|
|
+ * assigned as the queue &prep_rq_fn (in which case &struct request
|
|
|
+ * automagically gets a tag assigned). Note that this function
|
|
|
+ * assumes that any type of request can be queued! if this is not
|
|
|
+ * true for your device, you must check the request type before
|
|
|
+ * calling this function. The request will also be removed from
|
|
|
+ * the request queue, so it's the drivers responsibility to readd
|
|
|
+ * it if it should need to be restarted for some reason.
|
|
|
+ *
|
|
|
+ * Notes:
|
|
|
+ * queue lock must be held.
|
|
|
+ **/
|
|
|
+int blk_queue_start_tag(struct request_queue *q, struct request *rq)
|
|
|
+{
|
|
|
+ struct blk_queue_tag *bqt = q->queue_tags;
|
|
|
+ unsigned max_depth;
|
|
|
+ int tag;
|
|
|
+
|
|
|
+ if (unlikely((rq->cmd_flags & REQ_QUEUED))) {
|
|
|
+ printk(KERN_ERR
|
|
|
+ "%s: request %p for device [%s] already tagged %d",
|
|
|
+ __func__, rq,
|
|
|
+ rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
|
|
|
+ BUG();
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Protect against shared tag maps, as we may not have exclusive
|
|
|
+ * access to the tag map.
|
|
|
+ *
|
|
|
+ * We reserve a few tags just for sync IO, since we don't want
|
|
|
+ * to starve sync IO on behalf of flooding async IO.
|
|
|
+ */
|
|
|
+ max_depth = bqt->max_depth;
|
|
|
+ if (!rq_is_sync(rq) && max_depth > 1) {
|
|
|
+ max_depth -= 2;
|
|
|
+ if (!max_depth)
|
|
|
+ max_depth = 1;
|
|
|
+ if (q->in_flight[BLK_RW_ASYNC] > max_depth)
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ do {
|
|
|
+ tag = find_first_zero_bit(bqt->tag_map, max_depth);
|
|
|
+ if (tag >= max_depth)
|
|
|
+ return 1;
|
|
|
+
|
|
|
+ } while (test_and_set_bit_lock(tag, bqt->tag_map));
|
|
|
+ /*
|
|
|
+ * We need lock ordering semantics given by test_and_set_bit_lock.
|
|
|
+ * See blk_queue_end_tag for details.
|
|
|
+ */
|
|
|
+
|
|
|
+ rq->cmd_flags |= REQ_QUEUED;
|
|
|
+ rq->tag = tag;
|
|
|
+ bqt->tag_index[tag] = rq;
|
|
|
+ blk_start_request(rq);
|
|
|
+ list_add(&rq->queuelist, &q->tag_busy_list);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_queue_start_tag);
|
|
|
+
|
|
|
+/**
|
|
|
+ * blk_queue_invalidate_tags - invalidate all pending tags
|
|
|
+ * @q: the request queue for the device
|
|
|
+ *
|
|
|
+ * Description:
|
|
|
+ * Hardware conditions may dictate a need to stop all pending requests.
|
|
|
+ * In this case, we will safely clear the block side of the tag queue and
|
|
|
+ * readd all requests to the request queue in the right order.
|
|
|
+ *
|
|
|
+ * Notes:
|
|
|
+ * queue lock must be held.
|
|
|
+ **/
|
|
|
+void blk_queue_invalidate_tags(struct request_queue *q)
|
|
|
+{
|
|
|
+ struct list_head *tmp, *n;
|
|
|
+
|
|
|
+ list_for_each_safe(tmp, n, &q->tag_busy_list)
|
|
|
+ blk_requeue_request(q, list_entry_rq(tmp));
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(blk_queue_invalidate_tags);
|