1f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/module.h> 2fc1bc35443741e132dd0118e8dbac53f69a6f76eMatias Bjørling 3f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/moduleparam.h> 4f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/sched.h> 5f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/fs.h> 6f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/blkdev.h> 7f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/init.h> 8f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/slab.h> 9f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/blk-mq.h> 10f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe#include <linux/hrtimer.h> 11f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 12f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestruct nullb_cmd { 13f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct list_head list; 14f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct llist_node ll_list; 15f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct call_single_data csd; 16f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct request *rq; 17f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct bio *bio; 18f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int tag; 19f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_queue *nq; 20f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe}; 21f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 22f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestruct nullb_queue { 23f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned long *tag_map; 24f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe wait_queue_head_t wait; 25f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int queue_depth; 26f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 27f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmds; 28f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe}; 29f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 30f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestruct nullb { 31f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct list_head list; 32f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int index; 33f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct request_queue *q; 34f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct gendisk *disk; 3524d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig struct blk_mq_tag_set tag_set; 36f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct hrtimer timer; 37f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int queue_depth; 38f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe spinlock_t lock; 39f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 40f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_queue *queues; 41f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int nr_queues; 42f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe}; 43f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 44f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic LIST_HEAD(nullb_list); 45f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic struct mutex lock; 46f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int null_major; 47f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int nullb_indexes; 48f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 49f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestruct completion_queue { 50f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct llist_head list; 51f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct hrtimer timer; 52f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe}; 53f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 54f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe/* 55f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe * These are per-cpu for now, they will need to be configured by the 56f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe * complete_queues parameter and appropriately mapped. 57f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe */ 58f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic DEFINE_PER_CPU(struct completion_queue, completion_queues); 59f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 60f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboeenum { 61f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe NULL_IRQ_NONE = 0, 62f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe NULL_IRQ_SOFTIRQ = 1, 63f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe NULL_IRQ_TIMER = 2, 64ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig}; 65f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 66ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwigenum { 67f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe NULL_Q_BIO = 0, 68f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe NULL_Q_RQ = 1, 69f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe NULL_Q_MQ = 2, 70f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe}; 71f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 722d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorlingstatic int submit_queues; 73f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(submit_queues, int, S_IRUGO); 74f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(submit_queues, "Number of submission queues"); 75f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 76f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int home_node = NUMA_NO_NODE; 77f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(home_node, int, S_IRUGO); 78f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(home_node, "Home node for the device"); 79f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 80f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int queue_mode = NULL_Q_MQ; 81f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(queue_mode, int, S_IRUGO); 8254ae81cd5a20d23fdc7906a59d9018e1a760cadcMike SnitzerMODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)"); 83f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 84f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int gb = 250; 85f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(gb, int, S_IRUGO); 86f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(gb, "Size in GB"); 87f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 88f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int bs = 512; 89f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(bs, int, S_IRUGO); 90f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(bs, "Block size (in bytes)"); 91f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 92f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int nr_devices = 2; 93f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(nr_devices, int, S_IRUGO); 94f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(nr_devices, "Number of devices to register"); 95f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 96f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int irqmode = NULL_IRQ_SOFTIRQ; 97f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(irqmode, int, S_IRUGO); 98f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer"); 99f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 100f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int completion_nsec = 10000; 101f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(completion_nsec, int, S_IRUGO); 102f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns"); 103f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 104f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int hw_queue_depth = 64; 105f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(hw_queue_depth, int, S_IRUGO); 106f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64"); 107f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 108200052440d3b56f593038a35b7c14bdc780184a9Matias Bjørlingstatic bool use_per_node_hctx = false; 109f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_param(use_per_node_hctx, bool, S_IRUGO); 110200052440d3b56f593038a35b7c14bdc780184a9Matias BjørlingMODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false"); 111f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 112f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void put_tag(struct nullb_queue *nq, unsigned int tag) 113f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 114f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe clear_bit_unlock(tag, nq->tag_map); 115f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 116f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (waitqueue_active(&nq->wait)) 117f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe wake_up(&nq->wait); 118f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 119f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 120f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic unsigned int get_tag(struct nullb_queue *nq) 121f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 122f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int tag; 123f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 124f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe do { 125f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe tag = find_first_zero_bit(nq->tag_map, nq->queue_depth); 126f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (tag >= nq->queue_depth) 127f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return -1U; 128f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } while (test_and_set_bit_lock(tag, nq->tag_map)); 129f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 130f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return tag; 131f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 132f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 133f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void free_cmd(struct nullb_cmd *cmd) 134f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 135f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe put_tag(cmd->nq, cmd->tag); 136f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 137f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 138f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq) 139f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 140f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmd; 141f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int tag; 142f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 143f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe tag = get_tag(nq); 144f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (tag != -1U) { 145f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd = &nq->cmds[tag]; 146f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->tag = tag; 147f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->nq = nq; 148f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return cmd; 149f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 150f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 151f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return NULL; 152f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 153f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 154f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait) 155f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 156f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmd; 157f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe DEFINE_WAIT(wait); 158f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 159f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd = __alloc_cmd(nq); 160f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (cmd || !can_wait) 161f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return cmd; 162f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 163f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe do { 164f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE); 165f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd = __alloc_cmd(nq); 166f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (cmd) 167f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe break; 168f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 169f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe io_schedule(); 170f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } while (1); 171f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 172f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe finish_wait(&nq->wait, &wait); 173f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return cmd; 174f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 175f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 176f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void end_cmd(struct nullb_cmd *cmd) 177f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 178ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig switch (queue_mode) { 179ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig case NULL_Q_MQ: 180c8a446ad695ada43a885ec12b38411dbd190a11bChristoph Hellwig blk_mq_end_request(cmd->rq, 0); 181ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig return; 182ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig case NULL_Q_RQ: 183ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig INIT_LIST_HEAD(&cmd->rq->queuelist); 184ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig blk_end_request_all(cmd->rq, 0); 185ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig break; 186ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig case NULL_Q_BIO: 187f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe bio_endio(cmd->bio, 0); 188ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig break; 189ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig } 190f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 191ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig free_cmd(cmd); 192f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 193f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 194f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer) 195f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 196f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct completion_queue *cq; 197f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct llist_node *entry; 198f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmd; 199f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 200f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cq = &per_cpu(completion_queues, smp_processor_id()); 201f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 202f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe while ((entry = llist_del_all(&cq->list)) != NULL) { 203d7790b928d42597b7da21a4e43080774903e3b5cShlomo Pongratz entry = llist_reverse_order(entry); 204f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe do { 205f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd = container_of(entry, struct nullb_cmd, ll_list); 206f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe entry = entry->next; 207fc27691f3537a0df087214322467b642d1f6dedbMing Lei end_cmd(cmd); 208f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } while (entry); 209f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 210f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 211f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return HRTIMER_NORESTART; 212f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 213f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 214f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void null_cmd_end_timer(struct nullb_cmd *cmd) 215f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 216f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct completion_queue *cq = &per_cpu(completion_queues, get_cpu()); 217f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 218f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->ll_list.next = NULL; 219f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (llist_add(&cmd->ll_list, &cq->list)) { 220f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe ktime_t kt = ktime_set(0, completion_nsec); 221f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 222f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe hrtimer_start(&cq->timer, kt, HRTIMER_MODE_REL); 223f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 224f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 225f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe put_cpu(); 226f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 227f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 228f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void null_softirq_done_fn(struct request *rq) 229f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 230d891fa70876b37941a5c5bed813e73beb53ebcf7Jens Axboe if (queue_mode == NULL_Q_MQ) 231d891fa70876b37941a5c5bed813e73beb53ebcf7Jens Axboe end_cmd(blk_mq_rq_to_pdu(rq)); 232d891fa70876b37941a5c5bed813e73beb53ebcf7Jens Axboe else 233d891fa70876b37941a5c5bed813e73beb53ebcf7Jens Axboe end_cmd(rq->special); 234f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 235f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 236f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic inline void null_handle_cmd(struct nullb_cmd *cmd) 237f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 238f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe /* Complete IO by inline, softirq or timer */ 239f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe switch (irqmode) { 240f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe case NULL_IRQ_SOFTIRQ: 241ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig switch (queue_mode) { 242ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig case NULL_Q_MQ: 243ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig blk_mq_complete_request(cmd->rq); 244ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig break; 245ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig case NULL_Q_RQ: 246ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig blk_complete_request(cmd->rq); 247ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig break; 248ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig case NULL_Q_BIO: 249ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig /* 250ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig * XXX: no proper submitting cpu information available. 251ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig */ 252ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig end_cmd(cmd); 253ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig break; 254ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig } 255ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig break; 256ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig case NULL_IRQ_NONE: 257f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe end_cmd(cmd); 258f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe break; 259f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe case NULL_IRQ_TIMER: 260f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe null_cmd_end_timer(cmd); 261f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe break; 262f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 263f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 264f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 265f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic struct nullb_queue *nullb_to_queue(struct nullb *nullb) 266f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 267f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe int index = 0; 268f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 269f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (nullb->nr_queues != 1) 270f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues); 271f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 272f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return &nullb->queues[index]; 273f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 274f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 275f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void null_queue_bio(struct request_queue *q, struct bio *bio) 276f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 277f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb *nullb = q->queuedata; 278f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_queue *nq = nullb_to_queue(nullb); 279f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmd; 280f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 281f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd = alloc_cmd(nq, 1); 282f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->bio = bio; 283f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 284f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe null_handle_cmd(cmd); 285f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 286f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 287f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int null_rq_prep_fn(struct request_queue *q, struct request *req) 288f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 289f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb *nullb = q->queuedata; 290f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_queue *nq = nullb_to_queue(nullb); 291f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmd; 292f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 293f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd = alloc_cmd(nq, 0); 294f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (cmd) { 295f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->rq = req; 296f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe req->special = cmd; 297f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return BLKPREP_OK; 298f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 299f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 300f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return BLKPREP_DEFER; 301f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 302f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 303f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void null_request_fn(struct request_queue *q) 304f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 305f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct request *rq; 306f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 307f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe while ((rq = blk_fetch_request(q)) != NULL) { 308f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmd = rq->special; 309f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 310f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe spin_unlock_irq(q->queue_lock); 311f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe null_handle_cmd(cmd); 312f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe spin_lock_irq(q->queue_lock); 313f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 314f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 315f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 316bf57229745f849e500ba69ff91e35bc8160a7373Christoph Hellwigstatic int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq, 317bf57229745f849e500ba69ff91e35bc8160a7373Christoph Hellwig bool last) 318f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 3199d74e25737d73e93ccddeb5a61bcd56b7b8eb57bChristoph Hellwig struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq); 320f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 321f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->rq = rq; 322f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->nq = hctx->driver_data; 323f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 324e2490073cd7c3d6f6ef6e029a208edd4d38efac4Christoph Hellwig blk_mq_start_request(rq); 325e2490073cd7c3d6f6ef6e029a208edd4d38efac4Christoph Hellwig 326f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe null_handle_cmd(cmd); 327f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return BLK_MQ_RQ_QUEUE_OK; 328f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 329f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 3302d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorlingstatic void null_init_queue(struct nullb *nullb, struct nullb_queue *nq) 3312d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling{ 3322d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling BUG_ON(!nullb); 3332d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling BUG_ON(!nq); 3342d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling 3352d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling init_waitqueue_head(&nq->wait); 3362d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling nq->queue_depth = nullb->queue_depth; 3372d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling} 3382d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling 339f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 340f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int index) 341f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 342f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb *nullb = data; 343f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_queue *nq = &nullb->queues[index]; 344f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 345f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe hctx->driver_data = nq; 3462d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling null_init_queue(nullb, nq); 3472d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling nullb->nr_queues++; 348f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 349f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return 0; 350f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 351f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 352f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic struct blk_mq_ops null_mq_ops = { 353f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe .queue_rq = null_queue_rq, 354f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe .map_queue = blk_mq_map_queue, 355f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe .init_hctx = null_init_hctx, 356ce2c350b2cfe5b5ca5023a6b1ec4d21821d39addChristoph Hellwig .complete = null_softirq_done_fn, 357f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe}; 358f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 359f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void null_del_dev(struct nullb *nullb) 360f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 361f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe list_del_init(&nullb->list); 362f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 363f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe del_gendisk(nullb->disk); 364518d00b7498c5894be94545848d55e5b9c55749eMing Lei blk_cleanup_queue(nullb->q); 36524d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig if (queue_mode == NULL_Q_MQ) 36624d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig blk_mq_free_tag_set(&nullb->tag_set); 367f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe put_disk(nullb->disk); 368f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe kfree(nullb); 369f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 370f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 371f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int null_open(struct block_device *bdev, fmode_t mode) 372f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 373f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return 0; 374f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 375f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 376f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void null_release(struct gendisk *disk, fmode_t mode) 377f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 378f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 379f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 380f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic const struct block_device_operations null_fops = { 381f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe .owner = THIS_MODULE, 382f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe .open = null_open, 383f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe .release = null_release, 384f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe}; 385f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 386f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int setup_commands(struct nullb_queue *nq) 387f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 388f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb_cmd *cmd; 389f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe int i, tag_size; 390f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 391f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL); 392f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (!nq->cmds) 3932d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling return -ENOMEM; 394f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 395f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG; 396f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL); 397f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (!nq->tag_map) { 398f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe kfree(nq->cmds); 3992d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling return -ENOMEM; 400f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 401f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 402f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe for (i = 0; i < nq->queue_depth; i++) { 403f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd = &nq->cmds[i]; 404f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe INIT_LIST_HEAD(&cmd->list); 405f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->ll_list.next = NULL; 406f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cmd->tag = -1U; 407f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 408f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 409f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return 0; 410f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 411f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 412f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void cleanup_queue(struct nullb_queue *nq) 413f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 414f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe kfree(nq->tag_map); 415f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe kfree(nq->cmds); 416f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 417f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 418f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void cleanup_queues(struct nullb *nullb) 419f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 420f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe int i; 421f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 422f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe for (i = 0; i < nullb->nr_queues; i++) 423f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cleanup_queue(&nullb->queues[i]); 424f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 425f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe kfree(nullb->queues); 426f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 427f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 428f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int setup_queues(struct nullb *nullb) 429f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 4302d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling nullb->queues = kzalloc(submit_queues * sizeof(struct nullb_queue), 4312d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling GFP_KERNEL); 432f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (!nullb->queues) 4332d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling return -ENOMEM; 434f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 435f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb->nr_queues = 0; 436f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb->queue_depth = hw_queue_depth; 437f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 4382d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling return 0; 4392d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling} 4402d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling 4412d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorlingstatic int init_driver_queues(struct nullb *nullb) 4422d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling{ 4432d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling struct nullb_queue *nq; 4442d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling int i, ret = 0; 445f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 446f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe for (i = 0; i < submit_queues; i++) { 447f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nq = &nullb->queues[i]; 4482d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling 4492d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling null_init_queue(nullb, nq); 4502d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling 4512d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling ret = setup_commands(nq); 4522d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling if (ret) 45331f9690e6eaf549f3e643f6a8f7dab84fd31997aJan Kara return ret; 454f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb->nr_queues++; 455f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 4562d263a7856cbaf26dd89b671e2161c4a49f8461bMatias Bjorling return 0; 457f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 458f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 459f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int null_add_dev(void) 460f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 461f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct gendisk *disk; 462f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb *nullb; 463f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe sector_t size; 464dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott int rv; 465f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 466f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node); 467dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott if (!nullb) { 468dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott rv = -ENOMEM; 46924d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig goto out; 470dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott } 471f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 472f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe spin_lock_init(&nullb->lock); 473f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 47457053d8c5c59562cac156513740c10b502a40968Matias Bjorling if (queue_mode == NULL_Q_MQ && use_per_node_hctx) 47557053d8c5c59562cac156513740c10b502a40968Matias Bjorling submit_queues = nr_online_nodes; 47657053d8c5c59562cac156513740c10b502a40968Matias Bjorling 477dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott rv = setup_queues(nullb); 478dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott if (rv) 47924d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig goto out_free_nullb; 480f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 481f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (queue_mode == NULL_Q_MQ) { 482cdef54dd85ad66e77262ea57796a3e81683dd5d6Christoph Hellwig nullb->tag_set.ops = &null_mq_ops; 48324d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig nullb->tag_set.nr_hw_queues = submit_queues; 48424d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig nullb->tag_set.queue_depth = hw_queue_depth; 48524d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig nullb->tag_set.numa_node = home_node; 48624d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig nullb->tag_set.cmd_size = sizeof(struct nullb_cmd); 48724d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig nullb->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 48824d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig nullb->tag_set.driver_data = nullb; 48924d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig 490dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott rv = blk_mq_alloc_tag_set(&nullb->tag_set); 491dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott if (rv) 49224d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig goto out_cleanup_queues; 49324d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig 49424d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig nullb->q = blk_mq_init_queue(&nullb->tag_set); 495dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott if (!nullb->q) { 496dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott rv = -ENOMEM; 49724d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig goto out_cleanup_tags; 498dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott } 499f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } else if (queue_mode == NULL_Q_BIO) { 500f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb->q = blk_alloc_queue_node(GFP_KERNEL, home_node); 501dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott if (!nullb->q) { 502dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott rv = -ENOMEM; 50324d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig goto out_cleanup_queues; 504dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott } 505f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe blk_queue_make_request(nullb->q, null_queue_bio); 50631f9690e6eaf549f3e643f6a8f7dab84fd31997aJan Kara rv = init_driver_queues(nullb); 50731f9690e6eaf549f3e643f6a8f7dab84fd31997aJan Kara if (rv) 50831f9690e6eaf549f3e643f6a8f7dab84fd31997aJan Kara goto out_cleanup_blk_queue; 509f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } else { 510f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, home_node); 511dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott if (!nullb->q) { 512dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott rv = -ENOMEM; 51324d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig goto out_cleanup_queues; 514dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott } 515f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe blk_queue_prep_rq(nullb->q, null_rq_prep_fn); 51624d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig blk_queue_softirq_done(nullb->q, null_softirq_done_fn); 51731f9690e6eaf549f3e643f6a8f7dab84fd31997aJan Kara rv = init_driver_queues(nullb); 51831f9690e6eaf549f3e643f6a8f7dab84fd31997aJan Kara if (rv) 51931f9690e6eaf549f3e643f6a8f7dab84fd31997aJan Kara goto out_cleanup_blk_queue; 520f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 521f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 522f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb->q->queuedata = nullb; 523f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q); 524b277da0a8a594308e17881f4926879bd5fca2a2dMike Snitzer queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q); 525f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 526f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe disk = nullb->disk = alloc_disk_node(1, home_node); 527dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott if (!disk) { 528dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott rv = -ENOMEM; 52924d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig goto out_cleanup_blk_queue; 530dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott } 531f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 532f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe mutex_lock(&lock); 533f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe list_add_tail(&nullb->list, &nullb_list); 534f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb->index = nullb_indexes++; 535f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe mutex_unlock(&lock); 536f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 537f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe blk_queue_logical_block_size(nullb->q, bs); 538f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe blk_queue_physical_block_size(nullb->q, bs); 539f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 540f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe size = gb * 1024 * 1024 * 1024ULL; 541f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe sector_div(size, bs); 542f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe set_capacity(disk, size); 543f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 544f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe disk->flags |= GENHD_FL_EXT_DEVT; 545f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe disk->major = null_major; 546f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe disk->first_minor = nullb->index; 547f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe disk->fops = &null_fops; 548f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe disk->private_data = nullb; 549f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe disk->queue = nullb->q; 550f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe sprintf(disk->disk_name, "nullb%d", nullb->index); 551f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe add_disk(disk); 552f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return 0; 55324d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig 55424d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwigout_cleanup_blk_queue: 55524d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig blk_cleanup_queue(nullb->q); 55624d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwigout_cleanup_tags: 55724d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig if (queue_mode == NULL_Q_MQ) 55824d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig blk_mq_free_tag_set(&nullb->tag_set); 55924d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwigout_cleanup_queues: 56024d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig cleanup_queues(nullb); 56124d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwigout_free_nullb: 56224d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwig kfree(nullb); 56324d2f90309b23f2cfe016b2aebc5f0d6e01c57fdChristoph Hellwigout: 564dc501dc0d9dc9cbabc18b920f91a26c207e9476cRobert Elliott return rv; 565f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 566f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 567f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic int __init null_init(void) 568f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 569f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unsigned int i; 570f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 5719967d8ac5c0f3e730a21abefe4c889c724f3b212Raghavendra K T if (bs > PAGE_SIZE) { 5729967d8ac5c0f3e730a21abefe4c889c724f3b212Raghavendra K T pr_warn("null_blk: invalid block size\n"); 5739967d8ac5c0f3e730a21abefe4c889c724f3b212Raghavendra K T pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE); 5749967d8ac5c0f3e730a21abefe4c889c724f3b212Raghavendra K T bs = PAGE_SIZE; 5759967d8ac5c0f3e730a21abefe4c889c724f3b212Raghavendra K T } 576f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 577d15ee6b1a43afbe1a6cece3bd8d336a9d5cb7060Matias Bjorling if (queue_mode == NULL_Q_MQ && use_per_node_hctx) { 578fc1bc35443741e132dd0118e8dbac53f69a6f76eMatias Bjørling if (submit_queues < nr_online_nodes) { 579d15ee6b1a43afbe1a6cece3bd8d336a9d5cb7060Matias Bjorling pr_warn("null_blk: submit_queues param is set to %u.", 580d15ee6b1a43afbe1a6cece3bd8d336a9d5cb7060Matias Bjorling nr_online_nodes); 581fc1bc35443741e132dd0118e8dbac53f69a6f76eMatias Bjørling submit_queues = nr_online_nodes; 582fc1bc35443741e132dd0118e8dbac53f69a6f76eMatias Bjørling } 583d15ee6b1a43afbe1a6cece3bd8d336a9d5cb7060Matias Bjorling } else if (submit_queues > nr_cpu_ids) 584f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe submit_queues = nr_cpu_ids; 585f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe else if (!submit_queues) 586f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe submit_queues = 1; 587f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 588f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe mutex_init(&lock); 589f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 590f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe /* Initialize a separate list for each CPU for issuing softirqs */ 591f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe for_each_possible_cpu(i) { 592f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct completion_queue *cq = &per_cpu(completion_queues, i); 593f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 594f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe init_llist_head(&cq->list); 595f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 596f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (irqmode != NULL_IRQ_TIMER) 597f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe continue; 598f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 599f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe hrtimer_init(&cq->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 600f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe cq->timer.function = null_cmd_timer_expired; 601f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 602f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 603f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe null_major = register_blkdev(0, "nullb"); 604f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (null_major < 0) 605f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return null_major; 606f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 607f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe for (i = 0; i < nr_devices; i++) { 608f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe if (null_add_dev()) { 609f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unregister_blkdev(null_major, "nullb"); 610f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return -EINVAL; 611f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 612f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 613f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 614f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe pr_info("null: module loaded\n"); 615f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe return 0; 616f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 617f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 618f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboestatic void __exit null_exit(void) 619f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe{ 620f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe struct nullb *nullb; 621f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 622f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe unregister_blkdev(null_major, "nullb"); 623f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 624f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe mutex_lock(&lock); 625f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe while (!list_empty(&nullb_list)) { 626f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe nullb = list_entry(nullb_list.next, struct nullb, list); 627f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe null_del_dev(nullb); 628f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe } 629f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe mutex_unlock(&lock); 630f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe} 631f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 632f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_init(null_init); 633f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboemodule_exit(null_exit); 634f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens Axboe 635f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>"); 636f2298c0403b0dfcaef637eba0c02c4a06d7a25abJens AxboeMODULE_LICENSE("GPL"); 637