123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335 |
- /*
- * Copyright (C) 1991, 1992 Linus Torvalds
- * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
- * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
- * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
- * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
- * - July2000
- * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
- */
- /*
- * This handles all read/write requests to block devices
- */
- #include <linux/kernel.h>
- #include <linux/module.h>
- #include <linux/backing-dev.h>
- #include <linux/bio.h>
- #include <linux/blkdev.h>
- #include <linux/highmem.h>
- #include <linux/mm.h>
- #include <linux/kernel_stat.h>
- #include <linux/string.h>
- #include <linux/init.h>
- #include <linux/completion.h>
- #include <linux/slab.h>
- #include <linux/swap.h>
- #include <linux/writeback.h>
- #include <linux/task_io_accounting_ops.h>
- #include <linux/fault-inject.h>
- #include <linux/list_sort.h>
- #include <linux/delay.h>
- #include <linux/ratelimit.h>
- #define CREATE_TRACE_POINTS
- #include <trace/events/block.h>
- #include "blk.h"
- #include "blk-cgroup.h"
- EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
- EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
- EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
- EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
- DEFINE_IDA(blk_queue_ida);
- /*
- * For the allocated request tables
- */
- static struct kmem_cache *request_cachep;
- /*
- * For queue allocation
- */
- struct kmem_cache *blk_requestq_cachep;
- /*
- * Controlling structure to kblockd
- */
- static struct workqueue_struct *kblockd_workqueue;
- static void drive_stat_acct(struct request *rq, int new_io)
- {
- struct hd_struct *part;
- int rw = rq_data_dir(rq);
- int cpu;
- if (!blk_do_io_stat(rq))
- return;
- cpu = part_stat_lock();
- if (!new_io) {
- part = rq->part;
- part_stat_inc(cpu, part, merges[rw]);
- } else {
- part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
- if (!hd_struct_try_get(part)) {
- /*
- * The partition is already being removed,
- * the request will be accounted on the disk only
- *
- * We take a reference on disk->part0 although that
- * partition will never be deleted, so we can treat
- * it as any other partition.
- */
- part = &rq->rq_disk->part0;
- hd_struct_get(part);
- }
- part_round_stats(cpu, part);
- part_inc_in_flight(part, rw);
- rq->part = part;
- }
- part_stat_unlock();
- }
- void blk_queue_congestion_threshold(struct request_queue *q)
- {
- int nr;
- nr = q->nr_requests - (q->nr_requests / 8) + 1;
- if (nr > q->nr_requests)
- nr = q->nr_requests;
- q->nr_congestion_on = nr;
- nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
- if (nr < 1)
- nr = 1;
- q->nr_congestion_off = nr;
- }
- /**
- * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
- * @bdev: device
- *
- * Locates the passed device's request queue and returns the address of its
- * backing_dev_info
- *
- * Will return NULL if the request queue cannot be located.
- */
- struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
- {
- struct backing_dev_info *ret = NULL;
- struct request_queue *q = bdev_get_queue(bdev);
- if (q)
- ret = &q->backing_dev_info;
- return ret;
- }
- EXPORT_SYMBOL(blk_get_backing_dev_info);
- void blk_rq_init(struct request_queue *q, struct request *rq)
- {
- memset(rq, 0, sizeof(*rq));
- INIT_LIST_HEAD(&rq->queuelist);
- INIT_LIST_HEAD(&rq->timeout_list);
- rq->cpu = -1;
- rq->q = q;
- rq->__sector = (sector_t) -1;
- INIT_HLIST_NODE(&rq->hash);
- RB_CLEAR_NODE(&rq->rb_node);
- rq->cmd = rq->__cmd;
- rq->cmd_len = BLK_MAX_CDB;
- rq->tag = -1;
- rq->ref_count = 1;
- rq->start_time = jiffies;
- set_start_time_ns(rq);
- rq->part = NULL;
- }
- EXPORT_SYMBOL(blk_rq_init);
- static void req_bio_endio(struct request *rq, struct bio *bio,
- unsigned int nbytes, int error)
- {
- if (error)
- clear_bit(BIO_UPTODATE, &bio->bi_flags);
- else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
- error = -EIO;
- if (unlikely(nbytes > bio->bi_size)) {
- printk(KERN_ERR "%s: want %u bytes done, %u left\n",
- __func__, nbytes, bio->bi_size);
- nbytes = bio->bi_size;
- }
- if (unlikely(rq->cmd_flags & REQ_QUIET))
- set_bit(BIO_QUIET, &bio->bi_flags);
- bio->bi_size -= nbytes;
- bio->bi_sector += (nbytes >> 9);
- if (bio_integrity(bio))
- bio_integrity_advance(bio, nbytes);
- /* don't actually finish bio if it's part of flush sequence */
- if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
- bio_endio(bio, error);
- }
- void blk_dump_rq_flags(struct request *rq, char *msg)
- {
- int bit;
- printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
- rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
- rq->cmd_flags);
- printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
- (unsigned long long)blk_rq_pos(rq),
- blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
- printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n",
- rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
- if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
- printk(KERN_INFO " cdb: ");
- for (bit = 0; bit < BLK_MAX_CDB; bit++)
- printk("%02x ", rq->cmd[bit]);
- printk("\n");
- }
- }
- EXPORT_SYMBOL(blk_dump_rq_flags);
- static void blk_delay_work(struct work_struct *work)
- {
- struct request_queue *q;
- q = container_of(work, struct request_queue, delay_work.work);
- spin_lock_irq(q->queue_lock);
- __blk_run_queue(q);
- spin_unlock_irq(q->queue_lock);
- }
- /**
- * blk_delay_queue - restart queueing after defined interval
- * @q: The &struct request_queue in question
- * @msecs: Delay in msecs
- *
- * Description:
- * Sometimes queueing needs to be postponed for a little while, to allow
- * resources to come back. This function will make sure that queueing is
- * restarted around the specified time. Queue lock must be held.
- */
- void blk_delay_queue(struct request_queue *q, unsigned long msecs)
- {
- if (likely(!blk_queue_dead(q)))
- queue_delayed_work(kblockd_workqueue, &q->delay_work,
- msecs_to_jiffies(msecs));
- }
- EXPORT_SYMBOL(blk_delay_queue);
- /**
- * blk_start_queue - restart a previously stopped queue
- * @q: The &struct request_queue in question
- *
- * Description:
- * blk_start_queue() will clear the stop flag on the queue, and call
- * the request_fn for the queue if it was in a stopped state when
- * entered. Also see blk_stop_queue(). Queue lock must be held.
- **/
- void blk_start_queue(struct request_queue *q)
- {
- WARN_ON(!irqs_disabled());
- queue_flag_clear(QUEUE_FLAG_STOPPED, q);
- __blk_run_queue(q);
- }
- EXPORT_SYMBOL(blk_start_queue);
- /**
- * blk_stop_queue - stop a queue
- * @q: The &struct request_queue in question
- *
- * Description:
- * The Linux block layer assumes that a block driver will consume all
- * entries on the request queue when the request_fn strategy is called.
- * Often this will not happen, because of hardware limitations (queue
- * depth settings). If a device driver gets a 'queue full' response,
- * or if it simply chooses not to queue more I/O at one point, it can
- * call this function to prevent the request_fn from being called until
- * the driver has signalled it's ready to go again. This happens by calling
- * blk_start_queue() to restart queue operations. Queue lock must be held.
- **/
- void blk_stop_queue(struct request_queue *q)
- {
- cancel_delayed_work(&q->delay_work);
- queue_flag_set(QUEUE_FLAG_STOPPED, q);
- }
- EXPORT_SYMBOL(blk_stop_queue);
- /**
- * blk_sync_queue - cancel any pending callbacks on a queue
- * @q: the queue
- *
- * Description:
- * The block layer may perform asynchronous callback activity
- * on a queue, such as calling the unplug function after a timeout.
- * A block device may call blk_sync_queue to ensure that any
- * such activity is cancelled, thus allowing it to release resources
- * that the callbacks might use. The caller must already have made sure
- * that its ->make_request_fn will not re-add plugging prior to calling
- * this function.
- *
- * This function does not cancel any asynchronous activity arising
- * out of elevator or throttling code. That would require elevaotor_exit()
- * and blkcg_exit_queue() to be called with queue lock initialized.
- *
- */
- void blk_sync_queue(struct request_queue *q)
- {
- del_timer_sync(&q->timeout);
- cancel_delayed_work_sync(&q->delay_work);
- }
- EXPORT_SYMBOL(blk_sync_queue);
- /**
- * __blk_run_queue_uncond - run a queue whether or not it has been stopped
- * @q: The queue to run
- *
- * Description:
- * Invoke request handling on a queue if there are any pending requests.
- * May be used to restart request handling after a request has completed.
- * This variant runs the queue whether or not the queue has been
- * stopped. Must be called with the queue lock held and interrupts
- * disabled. See also @blk_run_queue.
- */
- inline void __blk_run_queue_uncond(struct request_queue *q)
- {
- if (unlikely(blk_queue_dead(q)))
- return;
- /*
- * Some request_fn implementations, e.g. scsi_request_fn(), unlock
- * the queue lock internally. As a result multiple threads may be
- * running such a request function concurrently. Keep track of the
- * number of active request_fn invocations such that blk_drain_queue()
- * can wait until all these request_fn calls have finished.
- */
- q->request_fn_active++;
- q->request_fn(q);
- q->request_fn_active--;
- }
- /**
- * __blk_run_queue - run a single device queue
- * @q: The queue to run
- *
- * Description:
- * See @blk_run_queue. This variant must be called with the queue lock
- * held and interrupts disabled.
- */
- void __blk_run_queue(struct request_queue *q)
- {
- if (unlikely(blk_queue_stopped(q)))
|