blkdev.h 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _LINUX_BLKDEV_H
  3. #define _LINUX_BLKDEV_H
  4. #include <linux/sched.h>
  5. #include <linux/sched/clock.h>
  6. #ifdef CONFIG_BLOCK
  7. #include <linux/major.h>
  8. #include <linux/genhd.h>
  9. #include <linux/list.h>
  10. #include <linux/llist.h>
  11. #include <linux/timer.h>
  12. #include <linux/workqueue.h>
  13. #include <linux/pagemap.h>
  14. #include <linux/backing-dev-defs.h>
  15. #include <linux/wait.h>
  16. #include <linux/mempool.h>
  17. #include <linux/pfn.h>
  18. #include <linux/bio.h>
  19. #include <linux/stringify.h>
  20. #include <linux/gfp.h>
  21. #include <linux/bsg.h>
  22. #include <linux/smp.h>
  23. #include <linux/rcupdate.h>
  24. #include <linux/percpu-refcount.h>
  25. #include <linux/scatterlist.h>
  26. #include <linux/blkzoned.h>
  27. #include <linux/seqlock.h>
  28. #include <linux/u64_stats_sync.h>
  29. struct module;
  30. struct scsi_ioctl_command;
  31. struct request_queue;
  32. struct elevator_queue;
  33. struct blk_trace;
  34. struct request;
  35. struct sg_io_hdr;
  36. struct bsg_job;
  37. struct blkcg_gq;
  38. struct blk_flush_queue;
  39. struct pr_ops;
  40. struct rq_wb;
  41. struct blk_queue_stats;
  42. struct blk_stat_callback;
  43. #define BLKDEV_MIN_RQ 4
  44. #define BLKDEV_MAX_RQ 128 /* Default maximum */
  45. /* Must be consistent with blk_mq_poll_stats_bkt() */
  46. #define BLK_MQ_POLL_STATS_BKTS 16
  47. /*
  48. * Maximum number of blkcg policies allowed to be registered concurrently.
  49. * Defined here to simplify include dependency.
  50. */
  51. #define BLKCG_MAX_POLS 3
  52. typedef void (rq_end_io_fn)(struct request *, blk_status_t);
  53. #define BLK_RL_SYNCFULL (1U << 0)
  54. #define BLK_RL_ASYNCFULL (1U << 1)
  55. struct request_list {
  56. struct request_queue *q; /* the queue this rl belongs to */
  57. #ifdef CONFIG_BLK_CGROUP
  58. struct blkcg_gq *blkg; /* blkg this request pool belongs to */
  59. #endif
  60. /*
  61. * count[], starved[], and wait[] are indexed by
  62. * BLK_RW_SYNC/BLK_RW_ASYNC
  63. */
  64. int count[2];
  65. int starved[2];
  66. mempool_t *rq_pool;
  67. wait_queue_head_t wait[2];
  68. unsigned int flags;
  69. };
  70. /*
  71. * request flags */
  72. typedef __u32 __bitwise req_flags_t;
  73. /* elevator knows about this request */
  74. #define RQF_SORTED ((__force req_flags_t)(1 << 0))
  75. /* drive already may have started this one */
  76. #define RQF_STARTED ((__force req_flags_t)(1 << 1))
  77. /* uses tagged queueing */
  78. #define RQF_QUEUED ((__force req_flags_t)(1 << 2))
  79. /* may not be passed by ioscheduler */
  80. #define RQF_SOFTBARRIER ((__force req_flags_t)(1 << 3))
  81. /* request for flush sequence */
  82. #define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4))
  83. /* merge of different types, fail separately */
  84. #define RQF_MIXED_MERGE ((__force req_flags_t)(1 << 5))
  85. /* track inflight for MQ */
  86. #define RQF_MQ_INFLIGHT ((__force req_flags_t)(1 << 6))
  87. /* don't call prep for this one */
  88. #define RQF_DONTPREP ((__force req_flags_t)(1 << 7))
  89. /* set for "ide_preempt" requests and also for requests for which the SCSI
  90. "quiesce" state must be ignored. */
  91. #define RQF_PREEMPT ((__force req_flags_t)(1 << 8))
  92. /* contains copies of user pages */
  93. #define RQF_COPY_USER ((__force req_flags_t)(1 << 9))
  94. /* vaguely specified driver internal error. Ignored by the block layer */
  95. #define RQF_FAILED ((__force req_flags_t)(1 << 10))
  96. /* don't warn about errors */
  97. #define RQF_QUIET ((__force req_flags_t)(1 << 11))
  98. /* elevator private data attached */
  99. #define RQF_ELVPRIV ((__force req_flags_t)(1 << 12))
  100. /* account I/O stat */
  101. #define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
  102. /* request came from our alloc pool */
  103. #define RQF_ALLOCED ((__force req_flags_t)(1 << 14))
  104. /* runtime pm request */
  105. #define RQF_PM ((__force req_flags_t)(1 << 15))
  106. /* on IO scheduler merge hash */
  107. #define RQF_HASHED ((__force req_flags_t)(1 << 16))
  108. /* IO stats tracking on */
  109. #define RQF_STATS ((__force req_flags_t)(1 << 17))
  110. /* Look at ->special_vec for the actual data payload instead of the
  111. bio chain. */
  112. #define RQF_SPECIAL_PAYLOAD ((__force req_flags_t)(1 << 18))
  113. /* The per-zone write lock is held for this request */
  114. #define RQF_ZONE_WRITE_LOCKED ((__force req_flags_t)(1 << 19))
  115. /* timeout is expired */
  116. #define RQF_MQ_TIMEOUT_EXPIRED ((__force req_flags_t)(1 << 20))
  117. /* already slept for hybrid poll */
  118. #define RQF_MQ_POLL_SLEPT ((__force req_flags_t)(1 << 21))
  119. /* flags that prevent us from merging requests: */
  120. #define RQF_NOMERGE_FLAGS \
  121. (RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
  122. /*
  123. * Try to put the fields that are referenced together in the same cacheline.
  124. *
  125. * If you modify this structure, make sure to update blk_rq_init() and
  126. * especially blk_mq_rq_ctx_init() to take care of the added fields.
  127. */
  128. struct request {
  129. struct request_queue *q;
  130. struct blk_mq_ctx *mq_ctx;
  131. int cpu;
  132. unsigned int cmd_flags; /* op and common flags */
  133. req_flags_t rq_flags;
  134. int internal_tag;
  135. /* the following two fields are internal, NEVER access directly */
  136. unsigned int __data_len; /* total data len */
  137. int tag;
  138. sector_t __sector; /* sector cursor */
  139. struct bio *bio;
  140. struct bio *biotail;
  141. struct list_head queuelist;
  142. /*
  143. * The hash is used inside the scheduler, and killed once the
  144. * request reaches the dispatch list. The ipi_list is only used
  145. * to queue the request for softirq completion, which is long
  146. * after the request has been unhashed (and even removed from
  147. * the dispatch list).
  148. */
  149. union {
  150. struct hlist_node hash; /* merge hash */
  151. struct list_head ipi_list;
  152. };
  153. /*
  154. * The rb_node is only used inside the io scheduler, requests
  155. * are pruned when moved to the dispatch queue. So let the
  156. * completion_data share space with the rb_node.
  157. */
  158. union {
  159. struct rb_node rb_node; /* sort/lookup */
  160. struct bio_vec special_vec;
  161. void *completion_data;
  162. int error_count; /* for legacy drivers, don't use */
  163. };
  164. /*
  165. * Three pointers are available for the IO schedulers, if they need
  166. * more they have to dynamically allocate it. Flush requests are
  167. * never put on the IO scheduler. So let the flush fields share
  168. * space with the elevator data.
  169. */
  170. union {
  171. struct {
  172. struct io_cq *icq;
  173. void *priv[2];
  174. } elv;
  175. struct {
  176. unsigned int seq;
  177. struct list_head list;
  178. rq_end_io_fn *saved_end_io;
  179. } flush;
  180. };
  181. struct gendisk *rq_disk;
  182. struct hd_struct *part;
  183. /* Time that I/O was submitted to the kernel. */
  184. u64 start_time_ns;
  185. /* Time that I/O was submitted to the device. */
  186. u64 io_start_time_ns;
  187. #ifdef CONFIG_BLK_WBT
  188. unsigned short wbt_flags;
  189. #endif
  190. #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
  191. unsigned short throtl_size;
  192. #endif
  193. /*
  194. * Number of scatter-gather DMA addr+len pairs after
  195. * physical address coalescing is performed.
  196. */
  197. unsigned short nr_phys_segments;
  198. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  199. unsigned short nr_integrity_segments;
  200. #endif
  201. unsigned short write_hint;
  202. unsigned short ioprio;
  203. unsigned int timeout;
  204. void *special; /* opaque pointer available for LLD use */
  205. unsigned int extra_len; /* length of alignment and padding */
  206. /*
  207. * On blk-mq, the lower bits of ->gstate (generation number and
  208. * state) carry the MQ_RQ_* state value and the upper bits the
  209. * generation number which is monotonically incremented and used to
  210. * distinguish the reuse instances.
  211. *
  212. * ->gstate_seq allows updates to ->gstate and other fields
  213. * (currently ->deadline) during request start to be read
  214. * atomically from the timeout path, so that it can operate on a
  215. * coherent set of information.
  216. */
  217. seqcount_t gstate_seq;
  218. u64 gstate;
  219. /*
  220. * ->aborted_gstate is used by the timeout to claim a specific
  221. * recycle instance of this request. See blk_mq_timeout_work().
  222. */
  223. struct u64_stats_sync aborted_gstate_sync;
  224. u64 aborted_gstate;
  225. /* access through blk_rq_set_deadline, blk_rq_deadline */
  226. unsigned long __deadline;
  227. struct list_head timeout_list;
  228. union {
  229. struct __call_single_data csd;
  230. u64 fifo_time;
  231. };
  232. /*
  233. * completion callback.
  234. */
  235. rq_end_io_fn *end_io;
  236. void *end_io_data;
  237. /* for bidi */
  238. struct request *next_rq;
  239. #ifdef CONFIG_BLK_CGROUP
  240. struct request_list *rl; /* rl this rq is alloced from */
  241. #endif
  242. };
  243. static inline bool blk_op_is_scsi(unsigned int op)
  244. {
  245. return op == REQ_OP_SCSI_IN || op == REQ_OP_SCSI_OUT;
  246. }
  247. static inline bool blk_op_is_private(unsigned int op)
  248. {
  249. return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
  250. }
  251. static inline bool blk_rq_is_scsi(struct request *rq)
  252. {
  253. return blk_op_is_scsi(req_op(rq));
  254. }
  255. static inline bool blk_rq_is_private(struct request *rq)
  256. {
  257. return blk_op_is_private(req_op(rq));
  258. }
  259. static inline bool blk_rq_is_passthrough(struct request *rq)
  260. {
  261. return blk_rq_is_scsi(rq) || blk_rq_is_private(rq);
  262. }
  263. static inline bool bio_is_passthrough(struct bio *bio)
  264. {
  265. unsigned op = bio_op(bio);
  266. return blk_op_is_scsi(op) || blk_op_is_private(op);
  267. }
  268. static inline unsigned short req_get_ioprio(struct request *req)
  269. {
  270. return req->ioprio;
  271. }
  272. #include <linux/elevator.h>
  273. struct blk_queue_ctx;
  274. typedef void (request_fn_proc) (struct request_queue *q);
  275. typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
  276. typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t);
  277. typedef int (prep_rq_fn) (struct request_queue *, struct request *);
  278. typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
  279. struct bio_vec;
  280. typedef void (softirq_done_fn)(struct request *);
  281. typedef int (dma_drain_needed_fn)(struct request *);
  282. typedef int (lld_busy_fn) (struct request_queue *q);
  283. typedef int (bsg_job_fn) (struct bsg_job *);
  284. typedef int (init_rq_fn)(struct request_queue *, struct request *, gfp_t);
  285. typedef void (exit_rq_fn)(struct request_queue *, struct request *);
  286. enum blk_eh_timer_return {
  287. BLK_EH_NOT_HANDLED,
  288. BLK_EH_HANDLED,
  289. BLK_EH_RESET_TIMER,
  290. };
  291. typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);
  292. enum blk_queue_state {
  293. Queue_down,
  294. Queue_up,
  295. };
  296. struct blk_queue_tag {
  297. struct request **tag_index; /* map of busy tags */
  298. unsigned long *tag_map; /* bit map of free/busy tags */
  299. int max_depth; /* what we will send to device */
  300. int real_max_depth; /* what the array can hold */
  301. atomic_t refcnt; /* map can be shared */
  302. int alloc_policy; /* tag allocation policy */
  303. int next_tag; /* next tag */
  304. };
  305. #define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
  306. #define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
  307. #define BLK_SCSI_MAX_CMDS (256)
  308. #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
  309. /*
  310. * Zoned block device models (zoned limit).
  311. */
  312. enum blk_zoned_model {
  313. BLK_ZONED_NONE, /* Regular block device */
  314. BLK_ZONED_HA, /* Host-aware zoned block device */
  315. BLK_ZONED_HM, /* Host-managed zoned block device */
  316. };
  317. struct queue_limits {
  318. unsigned long bounce_pfn;
  319. unsigned long seg_boundary_mask;
  320. unsigned long virt_boundary_mask;
  321. unsigned int max_hw_sectors;
  322. unsigned int max_dev_sectors;
  323. unsigned int chunk_sectors;
  324. unsigned int max_sectors;
  325. unsigned int max_segment_size;
  326. unsigned int physical_block_size;
  327. unsigned int alignment_offset;
  328. unsigned int io_min;
  329. unsigned int io_opt;
  330. unsigned int max_discard_sectors;
  331. unsigned int max_hw_discard_sectors;
  332. unsigned int max_write_same_sectors;
  333. unsigned int max_write_zeroes_sectors;
  334. unsigned int discard_granularity;
  335. unsigned int discard_alignment;
  336. unsigned short logical_block_size;
  337. unsigned short max_segments;
  338. unsigned short max_integrity_segments;
  339. unsigned short max_discard_segments;
  340. unsigned char misaligned;
  341. unsigned char discard_misaligned;
  342. unsigned char cluster;
  343. unsigned char raid_partial_stripes_expensive;
  344. enum blk_zoned_model zoned;
  345. };
  346. #ifdef CONFIG_BLK_DEV_ZONED
  347. struct blk_zone_report_hdr {
  348. unsigned int nr_zones;
  349. u8 padding[60];
  350. };
  351. extern int blkdev_report_zones(struct block_device *bdev,
  352. sector_t sector, struct blk_zone *zones,
  353. unsigned int *nr_zones, gfp_t gfp_mask);
  354. extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
  355. sector_t nr_sectors, gfp_t gfp_mask);
  356. extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
  357. unsigned int cmd, unsigned long arg);
  358. extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
  359. unsigned int cmd, unsigned long arg);
  360. #else /* CONFIG_BLK_DEV_ZONED */
  361. static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
  362. fmode_t mode, unsigned int cmd,
  363. unsigned long arg)
  364. {
  365. return -ENOTTY;
  366. }
  367. static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
  368. fmode_t mode, unsigned int cmd,
  369. unsigned long arg)
  370. {
  371. return -ENOTTY;
  372. }
  373. #endif /* CONFIG_BLK_DEV_ZONED */
  374. struct request_queue {
  375. /*
  376. * Together with queue_head for cacheline sharing
  377. */
  378. struct list_head queue_head;
  379. struct request *last_merge;
  380. struct elevator_queue *elevator;
  381. int nr_rqs[2]; /* # allocated [a]sync rqs */
  382. int nr_rqs_elvpriv; /* # allocated rqs w/ elvpriv */
  383. atomic_t shared_hctx_restart;
  384. struct blk_queue_stats *stats;
  385. struct rq_wb *rq_wb;
  386. /*
  387. * If blkcg is not used, @q->root_rl serves all requests. If blkcg
  388. * is used, root blkg allocates from @q->root_rl and all other
  389. * blkgs from their own blkg->rl. Which one to use should be
  390. * determined using bio_request_list().
  391. */
  392. struct request_list root_rl;
  393. request_fn_proc *request_fn;
  394. make_request_fn *make_request_fn;
  395. poll_q_fn *poll_fn;
  396. prep_rq_fn *prep_rq_fn;
  397. unprep_rq_fn *unprep_rq_fn;
  398. softirq_done_fn *softirq_done_fn;
  399. rq_timed_out_fn *rq_timed_out_fn;
  400. dma_drain_needed_fn *dma_drain_needed;
  401. lld_busy_fn *lld_busy_fn;
  402. /* Called just after a request is allocated */
  403. init_rq_fn *init_rq_fn;
  404. /* Called just before a request is freed */
  405. exit_rq_fn *exit_rq_fn;
  406. /* Called from inside blk_get_request() */
  407. void (*initialize_rq_fn)(struct request *rq);
  408. const struct blk_mq_ops *mq_ops;
  409. unsigned int *mq_map;
  410. /* sw queues */
  411. struct blk_mq_ctx __percpu *queue_ctx;
  412. unsigned int nr_queues;
  413. unsigned int queue_depth;
  414. /* hw dispatch queues */
  415. struct blk_mq_hw_ctx **queue_hw_ctx;
  416. unsigned int nr_hw_queues;
  417. /*
  418. * Dispatch queue sorting
  419. */
  420. sector_t end_sector;
  421. struct request *boundary_rq;
  422. /*
  423. * Delayed queue handling
  424. */
  425. struct delayed_work delay_work;
  426. struct backing_dev_info *backing_dev_info;
  427. /*
  428. * The queue owner gets to use this for whatever they like.
  429. * ll_rw_blk doesn't touch it.
  430. */
  431. void *queuedata;
  432. /*
  433. * various queue flags, see QUEUE_* below
  434. */
  435. unsigned long queue_flags;
  436. /*
  437. * ida allocated id for this queue. Used to index queues from
  438. * ioctx.
  439. */
  440. int id;
  441. /*
  442. * queue needs bounce pages for pages above this limit
  443. */
  444. gfp_t bounce_gfp;
  445. /*
  446. * protects queue structures from reentrancy. ->__queue_lock should
  447. * _never_ be used directly, it is queue private. always use
  448. * ->queue_lock.
  449. */
  450. spinlock_t __queue_lock;
  451. spinlock_t *queue_lock;
  452. /*
  453. * queue kobject
  454. */
  455. struct kobject kobj;
  456. /*
  457. * mq queue kobject
  458. */
  459. struct kobject mq_kobj;
  460. #ifdef CONFIG_BLK_DEV_INTEGRITY
  461. struct blk_integrity integrity;
  462. #endif /* CONFIG_BLK_DEV_INTEGRITY */
  463. #ifdef CONFIG_PM
  464. struct device *dev;
  465. int rpm_status;
  466. unsigned int nr_pending;
  467. #endif
  468. /*
  469. * queue settings
  470. */
  471. unsigned long nr_requests; /* Max # of requests */
  472. unsigned int nr_congestion_on;
  473. unsigned int nr_congestion_off;
  474. unsigned int nr_batching;
  475. unsigned int dma_drain_size;
  476. void *dma_drain_buffer;
  477. unsigned int dma_pad_mask;
  478. unsigned int dma_alignment;
  479. struct blk_queue_tag *queue_tags;
  480. struct list_head tag_busy_list;
  481. unsigned int nr_sorted;
  482. unsigned int in_flight[2];
  483. /*
  484. * Number of active block driver functions for which blk_drain_queue()
  485. * must wait. Must be incremented around functions that unlock the
  486. * queue_lock internally, e.g. scsi_request_fn().
  487. */
  488. unsigned int request_fn_active;
  489. unsigned int rq_timeout;
  490. int poll_nsec;
  491. struct blk_stat_callback *poll_cb;
  492. struct blk_rq_stat poll_stat[BLK_MQ_POLL_STATS_BKTS];
  493. struct timer_list timeout;
  494. struct work_struct timeout_work;
  495. struct list_head timeout_list;
  496. struct list_head icq_list;
  497. #ifdef CONFIG_BLK_CGROUP
  498. DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
  499. struct blkcg_gq *root_blkg;
  500. struct list_head blkg_list;
  501. #endif
  502. struct queue_limits limits;
  503. /*
  504. * Zoned block device information for request dispatch control.
  505. * nr_zones is the total number of zones of the device. This is always
  506. * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
  507. * bits which indicates if a zone is conventional (bit clear) or
  508. * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
  509. * bits which indicates if a zone is write locked, that is, if a write
  510. * request targeting the zone was dispatched. All three fields are
  511. * initialized by the low level device driver (e.g. scsi/sd.c).
  512. * Stacking drivers (device mappers) may or may not initialize
  513. * these fields.
  514. *
  515. * Reads of this information must be protected with blk_queue_enter() /
  516. * blk_queue_exit(). Modifying this information is only allowed while
  517. * no requests are being processed. See also blk_mq_freeze_queue() and
  518. * blk_mq_unfreeze_queue().
  519. */
  520. unsigned int nr_zones;
  521. unsigned long *seq_zones_bitmap;
  522. unsigned long *seq_zones_wlock;
  523. /*
  524. * sg stuff
  525. */
  526. unsigned int sg_timeout;
  527. unsigned int sg_reserved_size;
  528. int node;
  529. #ifdef CONFIG_BLK_DEV_IO_TRACE
  530. struct blk_trace *blk_trace;
  531. struct mutex blk_trace_mutex;
  532. #endif
  533. /*
  534. * for flush operations
  535. */
  536. struct blk_flush_queue *fq;
  537. struct list_head requeue_list;
  538. spinlock_t requeue_lock;
  539. struct delayed_work requeue_work;
  540. struct mutex sysfs_lock;
  541. int bypass_depth;
  542. atomic_t mq_freeze_depth;
  543. #if defined(CONFIG_BLK_DEV_BSG)
  544. bsg_job_fn *bsg_job_fn;
  545. struct bsg_class_device bsg_dev;
  546. #endif
  547. #ifdef CONFIG_BLK_DEV_THROTTLING
  548. /* Throttle data */
  549. struct throtl_data *td;
  550. #endif
  551. struct rcu_head rcu_head;
  552. wait_queue_head_t mq_freeze_wq;
  553. struct percpu_ref q_usage_counter;
  554. struct list_head all_q_node;
  555. struct blk_mq_tag_set *tag_set;
  556. struct list_head tag_set_list;
  557. struct bio_set *bio_split;
  558. #ifdef CONFIG_BLK_DEBUG_FS
  559. struct dentry *debugfs_dir;
  560. struct dentry *sched_debugfs_dir;
  561. #endif
  562. bool mq_sysfs_init_done;
  563. size_t cmd_size;
  564. void *rq_alloc_data;
  565. struct work_struct release_work;
  566. #define BLK_MAX_WRITE_HINTS 5
  567. u64 write_hints[BLK_MAX_WRITE_HINTS];
  568. };
  569. #define QUEUE_FLAG_QUEUED 0 /* uses generic tag queueing */
  570. #define QUEUE_FLAG_STOPPED 1 /* queue is stopped */
  571. #define QUEUE_FLAG_DYING 2 /* queue being torn down */
  572. #define QUEUE_FLAG_BYPASS 3 /* act as dumb FIFO queue */
  573. #define QUEUE_FLAG_BIDI 4 /* queue supports bidi requests */
  574. #define QUEUE_FLAG_NOMERGES 5 /* disable merge attempts */
  575. #define QUEUE_FLAG_SAME_COMP 6 /* complete on same CPU-group */
  576. #define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */
  577. #define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */
  578. #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
  579. #define QUEUE_FLAG_IO_STAT 10 /* do IO stats */
  580. #define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */
  581. #define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */
  582. #define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */
  583. #define QUEUE_FLAG_SECERASE 14 /* supports secure erase */
  584. #define QUEUE_FLAG_SAME_FORCE 15 /* force complete on same CPU */
  585. #define QUEUE_FLAG_DEAD 16 /* queue tear-down finished */
  586. #define QUEUE_FLAG_INIT_DONE 17 /* queue is initialized */
  587. #define QUEUE_FLAG_NO_SG_MERGE 18 /* don't attempt to merge SG segments*/
  588. #define QUEUE_FLAG_POLL 19 /* IO polling enabled if set */
  589. #define QUEUE_FLAG_WC 20 /* Write back caching */
  590. #define QUEUE_FLAG_FUA 21 /* device supports FUA writes */
  591. #define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */
  592. #define QUEUE_FLAG_DAX 23 /* device supports DAX */
  593. #define QUEUE_FLAG_STATS 24 /* track rq completion times */
  594. #define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */
  595. #define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */
  596. #define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */
  597. #define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */
  598. #define QUEUE_FLAG_PREEMPT_ONLY 29 /* only process REQ_PREEMPT requests */
  599. #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
  600. (1 << QUEUE_FLAG_SAME_COMP) | \
  601. (1 << QUEUE_FLAG_ADD_RANDOM))
  602. #define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
  603. (1 << QUEUE_FLAG_SAME_COMP) | \
  604. (1 << QUEUE_FLAG_POLL))
  605. void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
  606. void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
  607. bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
  608. bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
  609. #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
  610. #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
  611. #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
  612. #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
  613. #define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
  614. #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
  615. #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
  616. #define blk_queue_noxmerges(q) \
  617. test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
  618. #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
  619. #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
  620. #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
  621. #define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
  622. #define blk_queue_secure_erase(q) \
  623. (test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
  624. #define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
  625. #define blk_queue_scsi_passthrough(q) \
  626. test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
  627. #define blk_noretry_request(rq) \
  628. ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
  629. REQ_FAILFAST_DRIVER))
  630. #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
  631. #define blk_queue_preempt_only(q) \
  632. test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags)
  633. #define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
  634. extern int blk_set_preempt_only(struct request_queue *q);
  635. extern void blk_clear_preempt_only(struct request_queue *q);
  636. static inline int queue_in_flight(struct request_queue *q)
  637. {
  638. return q->in_flight[0] + q->in_flight[1];
  639. }
  640. static inline bool blk_account_rq(struct request *rq)
  641. {
  642. return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
  643. }
  644. #define blk_rq_cpu_valid(rq) ((rq)->cpu != -1)
  645. #define blk_bidi_rq(rq) ((rq)->next_rq != NULL)
  646. /* rq->queuelist of dequeued request must be list_empty() */
  647. #define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist))
  648. #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
  649. #define rq_data_dir(rq) (op_is_write(req_op(rq)) ? WRITE : READ)
  650. /*
  651. * Driver can handle struct request, if it either has an old style
  652. * request_fn defined, or is blk-mq based.
  653. */
  654. static inline bool queue_is_rq_based(struct request_queue *q)
  655. {
  656. return q->request_fn || q->mq_ops;
  657. }
  658. static inline unsigned int blk_queue_cluster(struct request_queue *q)
  659. {
  660. return q->limits.cluster;
  661. }
  662. static inline enum blk_zoned_model
  663. blk_queue_zoned_model(struct request_queue *q)
  664. {
  665. return q->limits.zoned;
  666. }
  667. static inline bool blk_queue_is_zoned(struct request_queue *q)
  668. {
  669. switch (blk_queue_zoned_model(q)) {
  670. case BLK_ZONED_HA:
  671. case BLK_ZONED_HM:
  672. return true;
  673. default:
  674. return false;
  675. }
  676. }
  677. static inline unsigned int blk_queue_zone_sectors(struct request_queue *q)
  678. {
  679. return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
  680. }
  681. static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
  682. {
  683. return q->nr_zones;
  684. }
  685. static inline unsigned int blk_queue_zone_no(struct request_queue *q,
  686. sector_t sector)
  687. {
  688. if (!blk_queue_is_zoned(q))
  689. return 0;
  690. return sector >> ilog2(q->limits.chunk_sectors);
  691. }
  692. static inline bool blk_queue_zone_is_seq(struct request_queue *q,
  693. sector_t sector)
  694. {
  695. if (!blk_queue_is_zoned(q) || !q->seq_zones_bitmap)
  696. return false;
  697. return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
  698. }
  699. static inline bool rq_is_sync(struct request *rq)
  700. {
  701. return op_is_sync(rq->cmd_flags);
  702. }
  703. static inline bool blk_rl_full(struct request_list *rl, bool sync)
  704. {
  705. unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
  706. return rl->flags & flag;
  707. }
  708. static inline void blk_set_rl_full(struct request_list *rl, bool sync)
  709. {
  710. unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
  711. rl->flags |= flag;
  712. }
  713. static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
  714. {
  715. unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
  716. rl->flags &= ~flag;
  717. }
  718. static inline bool rq_mergeable(struct request *rq)
  719. {
  720. if (blk_rq_is_passthrough(rq))
  721. return false;
  722. if (req_op(rq) == REQ_OP_FLUSH)
  723. return false;
  724. if (req_op(rq) == REQ_OP_WRITE_ZEROES)
  725. return false;
  726. if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
  727. return false;
  728. if (rq->rq_flags & RQF_NOMERGE_FLAGS)
  729. return false;
  730. return true;
  731. }
  732. static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
  733. {
  734. if (bio_page(a) == bio_page(b) &&
  735. bio_offset(a) == bio_offset(b))
  736. return true;
  737. return false;
  738. }
  739. static inline unsigned int blk_queue_depth(struct request_queue *q)
  740. {
  741. if (q->queue_depth)
  742. return q->queue_depth;
  743. return q->nr_requests;
  744. }
  745. /*
  746. * q->prep_rq_fn return values
  747. */
  748. enum {
  749. BLKPREP_OK, /* serve it */
  750. BLKPREP_KILL, /* fatal error, kill, return -EIO */
  751. BLKPREP_DEFER, /* leave on queue */
  752. BLKPREP_INVALID, /* invalid command, kill, return -EREMOTEIO */
  753. };
  754. extern unsigned long blk_max_low_pfn, blk_max_pfn;
  755. /*
  756. * standard bounce addresses:
  757. *
  758. * BLK_BOUNCE_HIGH : bounce all highmem pages
  759. * BLK_BOUNCE_ANY : don't bounce anything
  760. * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary
  761. */
  762. #if BITS_PER_LONG == 32
  763. #define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
  764. #else
  765. #define BLK_BOUNCE_HIGH -1ULL
  766. #endif
  767. #define BLK_BOUNCE_ANY (-1ULL)
  768. #define BLK_BOUNCE_ISA (DMA_BIT_MASK(24))
  769. /*
  770. * default timeout for SG_IO if none specified
  771. */
  772. #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
  773. #define BLK_MIN_SG_TIMEOUT (7 * HZ)
  774. struct rq_map_data {
  775. struct page **pages;
  776. int page_order;
  777. int nr_entries;
  778. unsigned long offset;
  779. int null_mapped;
  780. int from_user;
  781. };
  782. struct req_iterator {
  783. struct bvec_iter iter;
  784. struct bio *bio;
  785. };
  786. /* This should not be used directly - use rq_for_each_segment */
  787. #define for_each_bio(_bio) \
  788. for (; _bio; _bio = _bio->bi_next)
  789. #define __rq_for_each_bio(_bio, rq) \
  790. if ((rq->bio)) \
  791. for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
  792. #define rq_for_each_segment(bvl, _rq, _iter) \
  793. __rq_for_each_bio(_iter.bio, _rq) \
  794. bio_for_each_segment(bvl, _iter.bio, _iter.iter)
  795. #define rq_iter_last(bvec, _iter) \
  796. (_iter.bio->bi_next == NULL && \
  797. bio_iter_last(bvec, _iter.iter))
  798. #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  799. # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
  800. #endif
  801. #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  802. extern void rq_flush_dcache_pages(struct request *rq);
  803. #else
  804. static inline void rq_flush_dcache_pages(struct request *rq)
  805. {
  806. }
  807. #endif
  808. extern int blk_register_queue(struct gendisk *disk);
  809. extern void blk_unregister_queue(struct gendisk *disk);
  810. extern blk_qc_t generic_make_request(struct bio *bio);
  811. extern blk_qc_t direct_make_request(struct bio *bio);
  812. extern void blk_rq_init(struct request_queue *q, struct request *rq);
  813. extern void blk_init_request_from_bio(struct request *req, struct bio *bio);
  814. extern void blk_put_request(struct request *);
  815. extern void __blk_put_request(struct request_queue *, struct request *);
  816. extern struct request *blk_get_request(struct request_queue *, unsigned int op,
  817. blk_mq_req_flags_t flags);
  818. extern void blk_requeue_request(struct request_queue *, struct request *);
  819. extern int blk_lld_busy(struct request_queue *q);
  820. extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
  821. struct bio_set *bs, gfp_t gfp_mask,
  822. int (*bio_ctr)(struct bio *, struct bio *, void *),
  823. void *data);
  824. extern void blk_rq_unprep_clone(struct request *rq);
  825. extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
  826. struct request *rq);
  827. extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
  828. extern void blk_delay_queue(struct request_queue *, unsigned long);
  829. extern void blk_queue_split(struct request_queue *, struct bio **);
  830. extern void blk_recount_segments(struct request_queue *, struct bio *);
  831. extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
  832. extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
  833. unsigned int, void __user *);
  834. extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
  835. unsigned int, void __user *);
  836. extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
  837. struct scsi_ioctl_command __user *);
  838. extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
  839. extern void blk_queue_exit(struct request_queue *q);
  840. extern void blk_start_queue(struct request_queue *q);
  841. extern void blk_start_queue_async(struct request_queue *q);
  842. extern void blk_stop_queue(struct request_queue *q);
  843. extern void blk_sync_queue(struct request_queue *q);
  844. extern void __blk_stop_queue(struct request_queue *q);
  845. extern void __blk_run_queue(struct request_queue *q);
  846. extern void __blk_run_queue_uncond(struct request_queue *q);
  847. extern void blk_run_queue(struct request_queue *);
  848. extern void blk_run_queue_async(struct request_queue *q);
  849. extern int blk_rq_map_user(struct request_queue *, struct request *,
  850. struct rq_map_data *, void __user *, unsigned long,
  851. gfp_t);
  852. extern int blk_rq_unmap_user(struct bio *);
  853. extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
  854. extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
  855. struct rq_map_data *, const struct iov_iter *,
  856. gfp_t);
  857. extern void blk_execute_rq(struct request_queue *, struct gendisk *,
  858. struct request *, int);
  859. extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
  860. struct request *, int, rq_end_io_fn *);
  861. int blk_status_to_errno(blk_status_t status);
  862. blk_status_t errno_to_blk_status(int errno);
  863. bool blk_poll(struct request_queue *q, blk_qc_t cookie);
  864. static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
  865. {
  866. return bdev->bd_disk->queue; /* this is never NULL */
  867. }
  868. /*
  869. * The basic unit of block I/O is a sector. It is used in a number of contexts
  870. * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
  871. * bytes. Variables of type sector_t represent an offset or size that is a
  872. * multiple of 512 bytes. Hence these two constants.
  873. */
  874. #ifndef SECTOR_SHIFT
  875. #define SECTOR_SHIFT 9
  876. #endif
  877. #ifndef SECTOR_SIZE
  878. #define SECTOR_SIZE (1 << SECTOR_SHIFT)
  879. #endif
  880. /*
  881. * blk_rq_pos() : the current sector
  882. * blk_rq_bytes() : bytes left in the entire request
  883. * blk_rq_cur_bytes() : bytes left in the current segment
  884. * blk_rq_err_bytes() : bytes left till the next error boundary
  885. * blk_rq_sectors() : sectors left in the entire request
  886. * blk_rq_cur_sectors() : sectors left in the current segment
  887. */
  888. static inline sector_t blk_rq_pos(const struct request *rq)
  889. {
  890. return rq->__sector;
  891. }
  892. static inline unsigned int blk_rq_bytes(const struct request *rq)
  893. {
  894. return rq->__data_len;
  895. }
  896. static inline int blk_rq_cur_bytes(const struct request *rq)
  897. {
  898. return rq->bio ? bio_cur_bytes(rq->bio) : 0;
  899. }
  900. extern unsigned int blk_rq_err_bytes(const struct request *rq);
  901. static inline unsigned int blk_rq_sectors(const struct request *rq)
  902. {
  903. return blk_rq_bytes(rq) >> SECTOR_SHIFT;
  904. }
  905. static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
  906. {
  907. return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
  908. }
  909. static inline unsigned int blk_rq_zone_no(struct request *rq)
  910. {
  911. return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
  912. }
  913. static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
  914. {
  915. return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
  916. }
  917. /*
  918. * Some commands like WRITE SAME have a payload or data transfer size which
  919. * is different from the size of the request. Any driver that supports such
  920. * commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
  921. * calculate the data transfer size.
  922. */
  923. static inline unsigned int blk_rq_payload_bytes(struct request *rq)
  924. {
  925. if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
  926. return rq->special_vec.bv_len;
  927. return blk_rq_bytes(rq);
  928. }
  929. static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
  930. int op)
  931. {
  932. if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
  933. return min(q->limits.max_discard_sectors,
  934. UINT_MAX >> SECTOR_SHIFT);
  935. if (unlikely(op == REQ_OP_WRITE_SAME))
  936. return q->limits.max_write_same_sectors;
  937. if (unlikely(op == REQ_OP_WRITE_ZEROES))
  938. return q->limits.max_write_zeroes_sectors;
  939. return q->limits.max_sectors;
  940. }
  941. /*
  942. * Return maximum size of a request at given offset. Only valid for
  943. * file system requests.
  944. */
  945. static inline unsigned int blk_max_size_offset(struct request_queue *q,
  946. sector_t offset)
  947. {
  948. if (!q->limits.chunk_sectors)
  949. return q->limits.max_sectors;
  950. return q->limits.chunk_sectors -
  951. (offset & (q->limits.chunk_sectors - 1));
  952. }
  953. static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
  954. sector_t offset)
  955. {
  956. struct request_queue *q = rq->q;
  957. if (blk_rq_is_passthrough(rq))
  958. return q->limits.max_hw_sectors;
  959. if (!q->limits.chunk_sectors ||
  960. req_op(rq) == REQ_OP_DISCARD ||
  961. req_op(rq) == REQ_OP_SECURE_ERASE)
  962. return blk_queue_get_max_sectors(q, req_op(rq));
  963. return min(blk_max_size_offset(q, offset),
  964. blk_queue_get_max_sectors(q, req_op(rq)));
  965. }
  966. static inline unsigned int blk_rq_count_bios(struct request *rq)
  967. {
  968. unsigned int nr_bios = 0;
  969. struct bio *bio;
  970. __rq_for_each_bio(bio, rq)
  971. nr_bios++;
  972. return nr_bios;
  973. }
  974. /*
  975. * Request issue related functions.
  976. */
  977. extern struct request *blk_peek_request(struct request_queue *q);
  978. extern void blk_start_request(struct request *rq);
  979. extern struct request *blk_fetch_request(struct request_queue *q);
  980. void blk_steal_bios(struct bio_list *list, struct request *rq);
  981. /*
  982. * Request completion related functions.
  983. *
  984. * blk_update_request() completes given number of bytes and updates
  985. * the request without completing it.
  986. *
  987. * blk_end_request() and friends. __blk_end_request() must be called
  988. * with the request queue spinlock acquired.
  989. *
  990. * Several drivers define their own end_request and call
  991. * blk_end_request() for parts of the original function.
  992. * This prevents code duplication in drivers.
  993. */
  994. extern bool blk_update_request(struct request *rq, blk_status_t error,
  995. unsigned int nr_bytes);
  996. extern void blk_finish_request(struct request *rq, blk_status_t error);
  997. extern bool blk_end_request(struct request *rq, blk_status_t error,
  998. unsigned int nr_bytes);
  999. extern void blk_end_request_all(struct request *rq, blk_status_t error);
  1000. extern bool __blk_end_request(struct request *rq, blk_status_t error,
  1001. unsigned int nr_bytes);
  1002. extern void __blk_end_request_all(struct request *rq, blk_status_t error);
  1003. extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
  1004. extern void blk_complete_request(struct request *);
  1005. extern void __blk_complete_request(struct request *);
  1006. extern void blk_abort_request(struct request *);
  1007. extern void blk_unprep_request(struct request *);
  1008. /*
  1009. * Access functions for manipulating queue properties
  1010. */
  1011. extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
  1012. spinlock_t *lock, int node_id);
  1013. extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
  1014. extern int blk_init_allocated_queue(struct request_queue *);
  1015. extern void blk_cleanup_queue(struct request_queue *);
  1016. extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
  1017. extern void blk_queue_bounce_limit(struct request_queue *, u64);
  1018. extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
  1019. extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
  1020. extern void blk_queue_max_segments(struct request_queue *, unsigned short);
  1021. extern void blk_queue_max_discard_segments(struct request_queue *,
  1022. unsigned short);
  1023. extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
  1024. extern void blk_queue_max_discard_sectors(struct request_queue *q,
  1025. unsigned int max_discard_sectors);
  1026. extern void blk_queue_max_write_same_sectors(struct request_queue *q,
  1027. unsigned int max_write_same_sectors);
  1028. extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
  1029. unsigned int max_write_same_sectors);
  1030. extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
  1031. extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
  1032. extern void blk_queue_alignment_offset(struct request_queue *q,
  1033. unsigned int alignment);
  1034. extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
  1035. extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
  1036. extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
  1037. extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
  1038. extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
  1039. extern void blk_set_default_limits(struct queue_limits *lim);
  1040. extern void blk_set_stacking_limits(struct queue_limits *lim);
  1041. extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
  1042. sector_t offset);
  1043. extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
  1044. sector_t offset);
  1045. extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
  1046. sector_t offset);
  1047. extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
  1048. extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
  1049. extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
  1050. extern int blk_queue_dma_drain(struct request_queue *q,
  1051. dma_drain_needed_fn *dma_drain_needed,
  1052. void *buf, unsigned int size);
  1053. extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
  1054. extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
  1055. extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
  1056. extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
  1057. extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
  1058. extern void blk_queue_dma_alignment(struct request_queue *, int);
  1059. extern void blk_queue_update_dma_alignment(struct request_queue *, int);
  1060. extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
  1061. extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
  1062. extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
  1063. extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
  1064. extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
  1065. /*
  1066. * Number of physical segments as sent to the device.
  1067. *
  1068. * Normally this is the number of discontiguous data segments sent by the
  1069. * submitter. But for data-less command like discard we might have no
  1070. * actual data segments submitted, but the driver might have to add it's
  1071. * own special payload. In that case we still return 1 here so that this
  1072. * special payload will be mapped.
  1073. */
  1074. static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
  1075. {
  1076. if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
  1077. return 1;
  1078. return rq->nr_phys_segments;
  1079. }
  1080. /*
  1081. * Number of discard segments (or ranges) the driver needs to fill in.
  1082. * Each discard bio merged into a request is counted as one segment.
  1083. */
  1084. static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
  1085. {
  1086. return max_t(unsigned short, rq->nr_phys_segments, 1);
  1087. }
  1088. extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
  1089. extern void blk_dump_rq_flags(struct request *, char *);
  1090. extern long nr_blockdev_pages(void);
  1091. bool __must_check blk_get_queue(struct request_queue *);
  1092. struct request_queue *blk_alloc_queue(gfp_t);
  1093. struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
  1094. spinlock_t *lock);
  1095. extern void blk_put_queue(struct request_queue *);
  1096. extern void blk_set_queue_dying(struct request_queue *);
  1097. /*
  1098. * block layer runtime pm functions
  1099. */
  1100. #ifdef CONFIG_PM
  1101. extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
  1102. extern int blk_pre_runtime_suspend(struct request_queue *q);
  1103. extern void blk_post_runtime_suspend(struct request_queue *q, int err);
  1104. extern void blk_pre_runtime_resume(struct request_queue *q);
  1105. extern void blk_post_runtime_resume(struct request_queue *q, int err);
  1106. extern void blk_set_runtime_active(struct request_queue *q);
  1107. #else
  1108. static inline void blk_pm_runtime_init(struct request_queue *q,
  1109. struct device *dev) {}
  1110. static inline int blk_pre_runtime_suspend(struct request_queue *q)
  1111. {
  1112. return -ENOSYS;
  1113. }
  1114. static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
  1115. static inline void blk_pre_runtime_resume(struct request_queue *q) {}
  1116. static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
  1117. static inline void blk_set_runtime_active(struct request_queue *q) {}
  1118. #endif
  1119. /*
  1120. * blk_plug permits building a queue of related requests by holding the I/O
  1121. * fragments for a short period. This allows merging of sequential requests
  1122. * into single larger request. As the requests are moved from a per-task list to
  1123. * the device's request_queue in a batch, this results in improved scalability
  1124. * as the lock contention for request_queue lock is reduced.
  1125. *
  1126. * It is ok not to disable preemption when adding the request to the plug list
  1127. * or when attempting a merge, because blk_schedule_flush_list() will only flush
  1128. * the plug list when the task sleeps by itself. For details, please see
  1129. * schedule() where blk_schedule_flush_plug() is called.
  1130. */
  1131. struct blk_plug {
  1132. struct list_head list; /* requests */
  1133. struct list_head mq_list; /* blk-mq requests */
  1134. struct list_head cb_list; /* md requires an unplug callback */
  1135. };
  1136. #define BLK_MAX_REQUEST_COUNT 16
  1137. #define BLK_PLUG_FLUSH_SIZE (128 * 1024)
  1138. struct blk_plug_cb;
  1139. typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
  1140. struct blk_plug_cb {
  1141. struct list_head list;
  1142. blk_plug_cb_fn callback;
  1143. void *data;
  1144. };
  1145. extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
  1146. void *data, int size);
  1147. extern void blk_start_plug(struct blk_plug *);
  1148. extern void blk_finish_plug(struct blk_plug *);
  1149. extern void blk_flush_plug_list(struct blk_plug *, bool);
  1150. static inline void blk_flush_plug(struct task_struct *tsk)
  1151. {
  1152. struct blk_plug *plug = tsk->plug;
  1153. if (plug)
  1154. blk_flush_plug_list(plug, false);
  1155. }
  1156. static inline void blk_schedule_flush_plug(struct task_struct *tsk)
  1157. {
  1158. struct blk_plug *plug = tsk->plug;
  1159. if (plug)
  1160. blk_flush_plug_list(plug, true);
  1161. }
  1162. static inline bool blk_needs_flush_plug(struct task_struct *tsk)
  1163. {
  1164. struct blk_plug *plug = tsk->plug;
  1165. return plug &&
  1166. (!list_empty(&plug->list) ||
  1167. !list_empty(&plug->mq_list) ||
  1168. !list_empty(&plug->cb_list));
  1169. }
  1170. /*
  1171. * tag stuff
  1172. */
  1173. extern int blk_queue_start_tag(struct request_queue *, struct request *);
  1174. extern struct request *blk_queue_find_tag(struct request_queue *, int);
  1175. extern void blk_queue_end_tag(struct request_queue *, struct request *);
  1176. extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int);
  1177. extern void blk_queue_free_tags(struct request_queue *);
  1178. extern int blk_queue_resize_tags(struct request_queue *, int);
  1179. extern void blk_queue_invalidate_tags(struct request_queue *);
  1180. extern struct blk_queue_tag *blk_init_tags(int, int);
  1181. extern void blk_free_tags(struct blk_queue_tag *);
  1182. static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
  1183. int tag)
  1184. {
  1185. if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
  1186. return NULL;
  1187. return bqt->tag_index[tag];
  1188. }
  1189. extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
  1190. extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
  1191. sector_t nr_sects, gfp_t gfp_mask, struct page *page);
  1192. #define BLKDEV_DISCARD_SECURE (1 << 0) /* issue a secure erase */
  1193. extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
  1194. sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
  1195. extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
  1196. sector_t nr_sects, gfp_t gfp_mask, int flags,
  1197. struct bio **biop);
  1198. #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */
  1199. #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */
  1200. extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
  1201. sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
  1202. unsigned flags);
  1203. extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
  1204. sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
  1205. static inline int sb_issue_discard(struct super_block *sb, sector_t block,
  1206. sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
  1207. {
  1208. return blkdev_issue_discard(sb->s_bdev,
  1209. block << (sb->s_blocksize_bits -
  1210. SECTOR_SHIFT),
  1211. nr_blocks << (sb->s_blocksize_bits -
  1212. SECTOR_SHIFT),
  1213. gfp_mask, flags);
  1214. }
  1215. static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
  1216. sector_t nr_blocks, gfp_t gfp_mask)
  1217. {
  1218. return blkdev_issue_zeroout(sb->s_bdev,
  1219. block << (sb->s_blocksize_bits -
  1220. SECTOR_SHIFT),
  1221. nr_blocks << (sb->s_blocksize_bits -
  1222. SECTOR_SHIFT),
  1223. gfp_mask, 0);
  1224. }
  1225. extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
  1226. enum blk_default_limits {
  1227. BLK_MAX_SEGMENTS = 128,
  1228. BLK_SAFE_MAX_SECTORS = 255,
  1229. BLK_DEF_MAX_SECTORS = 2560,
  1230. BLK_MAX_SEGMENT_SIZE = 65536,
  1231. BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
  1232. };
  1233. #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)
  1234. static inline unsigned long queue_segment_boundary(struct request_queue *q)
  1235. {
  1236. return q->limits.seg_boundary_mask;
  1237. }
  1238. static inline unsigned long queue_virt_boundary(struct request_queue *q)
  1239. {
  1240. return q->limits.virt_boundary_mask;
  1241. }
  1242. static inline unsigned int queue_max_sectors(struct request_queue *q)
  1243. {
  1244. return q->limits.max_sectors;
  1245. }
  1246. static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
  1247. {
  1248. return q->limits.max_hw_sectors;
  1249. }
  1250. static inline unsigned short queue_max_segments(struct request_queue *q)
  1251. {
  1252. return q->limits.max_segments;
  1253. }
  1254. static inline unsigned short queue_max_discard_segments(struct request_queue *q)
  1255. {
  1256. return q->limits.max_discard_segments;
  1257. }
  1258. static inline unsigned int queue_max_segment_size(struct request_queue *q)
  1259. {
  1260. return q->limits.max_segment_size;
  1261. }
  1262. static inline unsigned short queue_logical_block_size(struct request_queue *q)
  1263. {
  1264. int retval = 512;
  1265. if (q && q->limits.logical_block_size)
  1266. retval = q->limits.logical_block_size;
  1267. return retval;
  1268. }
  1269. static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
  1270. {
  1271. return queue_logical_block_size(bdev_get_queue(bdev));
  1272. }
  1273. static inline unsigned int queue_physical_block_size(struct request_queue *q)
  1274. {
  1275. return q->limits.physical_block_size;
  1276. }
  1277. static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
  1278. {
  1279. return queue_physical_block_size(bdev_get_queue(bdev));
  1280. }
  1281. static inline unsigned int queue_io_min(struct request_queue *q)
  1282. {
  1283. return q->limits.io_min;
  1284. }
  1285. static inline int bdev_io_min(struct block_device *bdev)
  1286. {
  1287. return queue_io_min(bdev_get_queue(bdev));
  1288. }
  1289. static inline unsigned int queue_io_opt(struct request_queue *q)
  1290. {
  1291. return q->limits.io_opt;
  1292. }
  1293. static inline int bdev_io_opt(struct block_device *bdev)
  1294. {
  1295. return queue_io_opt(bdev_get_queue(bdev));
  1296. }
  1297. static inline int queue_alignment_offset(struct request_queue *q)
  1298. {
  1299. if (q->limits.misaligned)
  1300. return -1;
  1301. return q->limits.alignment_offset;
  1302. }
  1303. static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
  1304. {
  1305. unsigned int granularity = max(lim->physical_block_size, lim->io_min);
  1306. unsigned int alignment = sector_div(sector, granularity >> SECTOR_SHIFT)
  1307. << SECTOR_SHIFT;
  1308. return (granularity + lim->alignment_offset - alignment) % granularity;
  1309. }
  1310. static inline int bdev_alignment_offset(struct block_device *bdev)
  1311. {
  1312. struct request_queue *q = bdev_get_queue(bdev);
  1313. if (q->limits.misaligned)
  1314. return -1;
  1315. if (bdev != bdev->bd_contains)
  1316. return bdev->bd_part->alignment_offset;
  1317. return q->limits.alignment_offset;
  1318. }
  1319. static inline int queue_discard_alignment(struct request_queue *q)
  1320. {
  1321. if (q->limits.discard_misaligned)
  1322. return -1;
  1323. return q->limits.discard_alignment;
  1324. }
  1325. static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
  1326. {
  1327. unsigned int alignment, granularity, offset;
  1328. if (!lim->max_discard_sectors)
  1329. return 0;
  1330. /* Why are these in bytes, not sectors? */
  1331. alignment = lim->discard_alignment >> SECTOR_SHIFT;
  1332. granularity = lim->discard_granularity >> SECTOR_SHIFT;
  1333. if (!granularity)
  1334. return 0;
  1335. /* Offset of the partition start in 'granularity' sectors */
  1336. offset = sector_div(sector, granularity);
  1337. /* And why do we do this modulus *again* in blkdev_issue_discard()? */
  1338. offset = (granularity + alignment - offset) % granularity;
  1339. /* Turn it back into bytes, gaah */
  1340. return offset << SECTOR_SHIFT;
  1341. }
  1342. static inline int bdev_discard_alignment(struct block_device *bdev)
  1343. {
  1344. struct request_queue *q = bdev_get_queue(bdev);
  1345. if (bdev != bdev->bd_contains)
  1346. return bdev->bd_part->discard_alignment;
  1347. return q->limits.discard_alignment;
  1348. }
  1349. static inline unsigned int bdev_write_same(struct block_device *bdev)
  1350. {
  1351. struct request_queue *q = bdev_get_queue(bdev);
  1352. if (q)
  1353. return q->limits.max_write_same_sectors;
  1354. return 0;
  1355. }
  1356. static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
  1357. {
  1358. struct request_queue *q = bdev_get_queue(bdev);
  1359. if (q)
  1360. return q->limits.max_write_zeroes_sectors;
  1361. return 0;
  1362. }
  1363. static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
  1364. {
  1365. struct request_queue *q = bdev_get_queue(bdev);
  1366. if (q)
  1367. return blk_queue_zoned_model(q);
  1368. return BLK_ZONED_NONE;
  1369. }
  1370. static inline bool bdev_is_zoned(struct block_device *bdev)
  1371. {
  1372. struct request_queue *q = bdev_get_queue(bdev);
  1373. if (q)
  1374. return blk_queue_is_zoned(q);
  1375. return false;
  1376. }
  1377. static inline unsigned int bdev_zone_sectors(struct block_device *bdev)
  1378. {
  1379. struct request_queue *q = bdev_get_queue(bdev);
  1380. if (q)
  1381. return blk_queue_zone_sectors(q);
  1382. return 0;
  1383. }
  1384. static inline unsigned int bdev_nr_zones(struct block_device *bdev)
  1385. {
  1386. struct request_queue *q = bdev_get_queue(bdev);
  1387. if (q)
  1388. return blk_queue_nr_zones(q);
  1389. return 0;
  1390. }
  1391. static inline int queue_dma_alignment(struct request_queue *q)
  1392. {
  1393. return q ? q->dma_alignment : 511;
  1394. }
  1395. static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
  1396. unsigned int len)
  1397. {
  1398. unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
  1399. return !(addr & alignment) && !(len & alignment);
  1400. }
  1401. /* assumes size > 256 */
  1402. static inline unsigned int blksize_bits(unsigned int size)
  1403. {
  1404. unsigned int bits = 8;
  1405. do {
  1406. bits++;
  1407. size >>= 1;
  1408. } while (size > 256);
  1409. return bits;
  1410. }
  1411. static inline unsigned int block_size(struct block_device *bdev)
  1412. {
  1413. return bdev->bd_block_size;
  1414. }
  1415. static inline bool queue_flush_queueable(struct request_queue *q)
  1416. {
  1417. return !test_bit(QUEUE_FLAG_FLUSH_NQ, &q->queue_flags);
  1418. }
  1419. typedef struct {struct page *v;} Sector;
  1420. unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
  1421. static inline void put_dev_sector(Sector p)
  1422. {
  1423. put_page(p.v);
  1424. }
  1425. static inline bool __bvec_gap_to_prev(struct request_queue *q,
  1426. struct bio_vec *bprv, unsigned int offset)
  1427. {
  1428. return offset ||
  1429. ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
  1430. }
  1431. /*
  1432. * Check if adding a bio_vec after bprv with offset would create a gap in
  1433. * the SG list. Most drivers don't care about this, but some do.
  1434. */
  1435. static inline bool bvec_gap_to_prev(struct request_queue *q,
  1436. struct bio_vec *bprv, unsigned int offset)
  1437. {
  1438. if (!queue_virt_boundary(q))
  1439. return false;
  1440. return __bvec_gap_to_prev(q, bprv, offset);
  1441. }
  1442. /*
  1443. * Check if the two bvecs from two bios can be merged to one segment.
  1444. * If yes, no need to check gap between the two bios since the 1st bio
  1445. * and the 1st bvec in the 2nd bio can be handled in one segment.
  1446. */
  1447. static inline bool bios_segs_mergeable(struct request_queue *q,
  1448. struct bio *prev, struct bio_vec *prev_last_bv,
  1449. struct bio_vec *next_first_bv)
  1450. {
  1451. if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv))
  1452. return false;
  1453. if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv))
  1454. return false;
  1455. if (prev->bi_seg_back_size + next_first_bv->bv_len >
  1456. queue_max_segment_size(q))
  1457. return false;
  1458. return true;
  1459. }
  1460. static inline bool bio_will_gap(struct request_queue *q,
  1461. struct request *prev_rq,
  1462. struct bio *prev,
  1463. struct bio *next)
  1464. {
  1465. if (bio_has_data(prev) && queue_virt_boundary(q)) {
  1466. struct bio_vec pb, nb;
  1467. /*
  1468. * don't merge if the 1st bio starts with non-zero
  1469. * offset, otherwise it is quite difficult to respect
  1470. * sg gap limit. We work hard to merge a huge number of small
  1471. * single bios in case of mkfs.
  1472. */
  1473. if (prev_rq)
  1474. bio_get_first_bvec(prev_rq->bio, &pb);
  1475. else
  1476. bio_get_first_bvec(prev, &pb);
  1477. if (pb.bv_offset)
  1478. return true;
  1479. /*
  1480. * We don't need to worry about the situation that the
  1481. * merged segment ends in unaligned virt boundary:
  1482. *
  1483. * - if 'pb' ends aligned, the merged segment ends aligned
  1484. * - if 'pb' ends unaligned, the next bio must include
  1485. * one single bvec of 'nb', otherwise the 'nb' can't
  1486. * merge with 'pb'
  1487. */
  1488. bio_get_last_bvec(prev, &pb);
  1489. bio_get_first_bvec(next, &nb);
  1490. if (!bios_segs_mergeable(q, prev, &pb, &nb))
  1491. return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
  1492. }
  1493. return false;
  1494. }
  1495. static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
  1496. {
  1497. return bio_will_gap(req->q, req, req->biotail, bio);
  1498. }
  1499. static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
  1500. {
  1501. return bio_will_gap(req->q, NULL, bio, req->bio);
  1502. }
  1503. int kblockd_schedule_work(struct work_struct *work);
  1504. int kblockd_schedule_work_on(int cpu, struct work_struct *work);
  1505. int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
  1506. #define MODULE_ALIAS_BLOCKDEV(major,minor) \
  1507. MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
  1508. #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
  1509. MODULE_ALIAS("block-major-" __stringify(major) "-*")
  1510. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  1511. enum blk_integrity_flags {
  1512. BLK_INTEGRITY_VERIFY = 1 << 0,
  1513. BLK_INTEGRITY_GENERATE = 1 << 1,
  1514. BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2,
  1515. BLK_INTEGRITY_IP_CHECKSUM = 1 << 3,
  1516. };
  1517. struct blk_integrity_iter {
  1518. void *prot_buf;
  1519. void *data_buf;
  1520. sector_t seed;
  1521. unsigned int data_size;
  1522. unsigned short interval;
  1523. const char *disk_name;
  1524. };
  1525. typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
  1526. struct blk_integrity_profile {
  1527. integrity_processing_fn *generate_fn;
  1528. integrity_processing_fn *verify_fn;
  1529. const char *name;
  1530. };
  1531. extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
  1532. extern void blk_integrity_unregister(struct gendisk *);
  1533. extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
  1534. extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
  1535. struct scatterlist *);
  1536. extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
  1537. extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
  1538. struct request *);
  1539. extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
  1540. struct bio *);
  1541. static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
  1542. {
  1543. struct blk_integrity *bi = &disk->queue->integrity;
  1544. if (!bi->profile)
  1545. return NULL;
  1546. return bi;
  1547. }
  1548. static inline
  1549. struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
  1550. {
  1551. return blk_get_integrity(bdev->bd_disk);
  1552. }
  1553. static inline bool blk_integrity_rq(struct request *rq)
  1554. {
  1555. return rq->cmd_flags & REQ_INTEGRITY;
  1556. }
  1557. static inline void blk_queue_max_integrity_segments(struct request_queue *q,
  1558. unsigned int segs)
  1559. {
  1560. q->limits.max_integrity_segments = segs;
  1561. }
  1562. static inline unsigned short
  1563. queue_max_integrity_segments(struct request_queue *q)
  1564. {
  1565. return q->limits.max_integrity_segments;
  1566. }
  1567. static inline bool integrity_req_gap_back_merge(struct request *req,
  1568. struct bio *next)
  1569. {
  1570. struct bio_integrity_payload *bip = bio_integrity(req->bio);
  1571. struct bio_integrity_payload *bip_next = bio_integrity(next);
  1572. return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
  1573. bip_next->bip_vec[0].bv_offset);
  1574. }
  1575. static inline bool integrity_req_gap_front_merge(struct request *req,
  1576. struct bio *bio)
  1577. {
  1578. struct bio_integrity_payload *bip = bio_integrity(bio);
  1579. struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
  1580. return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
  1581. bip_next->bip_vec[0].bv_offset);
  1582. }
  1583. #else /* CONFIG_BLK_DEV_INTEGRITY */
  1584. struct bio;
  1585. struct block_device;
  1586. struct gendisk;
  1587. struct blk_integrity;
  1588. static inline int blk_integrity_rq(struct request *rq)
  1589. {
  1590. return 0;
  1591. }
  1592. static inline int blk_rq_count_integrity_sg(struct request_queue *q,
  1593. struct bio *b)
  1594. {
  1595. return 0;
  1596. }
  1597. static inline int blk_rq_map_integrity_sg(struct request_queue *q,
  1598. struct bio *b,
  1599. struct scatterlist *s)
  1600. {
  1601. return 0;
  1602. }
  1603. static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
  1604. {
  1605. return NULL;
  1606. }
  1607. static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
  1608. {
  1609. return NULL;
  1610. }
  1611. static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
  1612. {
  1613. return 0;
  1614. }
  1615. static inline void blk_integrity_register(struct gendisk *d,
  1616. struct blk_integrity *b)
  1617. {
  1618. }
  1619. static inline void blk_integrity_unregister(struct gendisk *d)
  1620. {
  1621. }
  1622. static inline void blk_queue_max_integrity_segments(struct request_queue *q,
  1623. unsigned int segs)
  1624. {
  1625. }
  1626. static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
  1627. {
  1628. return 0;
  1629. }
  1630. static inline bool blk_integrity_merge_rq(struct request_queue *rq,
  1631. struct request *r1,
  1632. struct request *r2)
  1633. {
  1634. return true;
  1635. }
  1636. static inline bool blk_integrity_merge_bio(struct request_queue *rq,
  1637. struct request *r,
  1638. struct bio *b)
  1639. {
  1640. return true;
  1641. }
  1642. static inline bool integrity_req_gap_back_merge(struct request *req,
  1643. struct bio *next)
  1644. {
  1645. return false;
  1646. }
  1647. static inline bool integrity_req_gap_front_merge(struct request *req,
  1648. struct bio *bio)
  1649. {
  1650. return false;
  1651. }
  1652. #endif /* CONFIG_BLK_DEV_INTEGRITY */
  1653. struct block_device_operations {
  1654. int (*open) (struct block_device *, fmode_t);
  1655. void (*release) (struct gendisk *, fmode_t);
  1656. int (*rw_page)(struct block_device *, sector_t, struct page *, bool);
  1657. int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
  1658. int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
  1659. unsigned int (*check_events) (struct gendisk *disk,
  1660. unsigned int clearing);
  1661. /* ->media_changed() is DEPRECATED, use ->check_events() instead */
  1662. int (*media_changed) (struct gendisk *);
  1663. void (*unlock_native_capacity) (struct gendisk *);
  1664. int (*revalidate_disk) (struct gendisk *);
  1665. int (*getgeo)(struct block_device *, struct hd_geometry *);
  1666. /* this callback is with swap_lock and sometimes page table lock held */
  1667. void (*swap_slot_free_notify) (struct block_device *, unsigned long);
  1668. struct module *owner;
  1669. const struct pr_ops *pr_ops;
  1670. };
  1671. extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
  1672. unsigned long);
  1673. extern int bdev_read_page(struct block_device *, sector_t, struct page *);
  1674. extern int bdev_write_page(struct block_device *, sector_t, struct page *,
  1675. struct writeback_control *);
  1676. #ifdef CONFIG_BLK_DEV_ZONED
  1677. bool blk_req_needs_zone_write_lock(struct request *rq);
  1678. void __blk_req_zone_write_lock(struct request *rq);
  1679. void __blk_req_zone_write_unlock(struct request *rq);
  1680. static inline void blk_req_zone_write_lock(struct request *rq)
  1681. {
  1682. if (blk_req_needs_zone_write_lock(rq))
  1683. __blk_req_zone_write_lock(rq);
  1684. }
  1685. static inline void blk_req_zone_write_unlock(struct request *rq)
  1686. {
  1687. if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
  1688. __blk_req_zone_write_unlock(rq);
  1689. }
  1690. static inline bool blk_req_zone_is_write_locked(struct request *rq)
  1691. {
  1692. return rq->q->seq_zones_wlock &&
  1693. test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
  1694. }
  1695. static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
  1696. {
  1697. if (!blk_req_needs_zone_write_lock(rq))
  1698. return true;
  1699. return !blk_req_zone_is_write_locked(rq);
  1700. }
  1701. #else
  1702. static inline bool blk_req_needs_zone_write_lock(struct request *rq)
  1703. {
  1704. return false;
  1705. }
  1706. static inline void blk_req_zone_write_lock(struct request *rq)
  1707. {
  1708. }
  1709. static inline void blk_req_zone_write_unlock(struct request *rq)
  1710. {
  1711. }
  1712. static inline bool blk_req_zone_is_write_locked(struct request *rq)
  1713. {
  1714. return false;
  1715. }
  1716. static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
  1717. {
  1718. return true;
  1719. }
  1720. #endif /* CONFIG_BLK_DEV_ZONED */
  1721. #else /* CONFIG_BLOCK */
  1722. struct block_device;
  1723. /*
  1724. * stubs for when the block layer is configured out
  1725. */
  1726. #define buffer_heads_over_limit 0
  1727. static inline long nr_blockdev_pages(void)
  1728. {
  1729. return 0;
  1730. }
  1731. struct blk_plug {
  1732. };
  1733. static inline void blk_start_plug(struct blk_plug *plug)
  1734. {
  1735. }
  1736. static inline void blk_finish_plug(struct blk_plug *plug)
  1737. {
  1738. }
  1739. static inline void blk_flush_plug(struct task_struct *task)
  1740. {
  1741. }
  1742. static inline void blk_schedule_flush_plug(struct task_struct *task)
  1743. {
  1744. }
  1745. static inline bool blk_needs_flush_plug(struct task_struct *tsk)
  1746. {
  1747. return false;
  1748. }
  1749. static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
  1750. sector_t *error_sector)
  1751. {
  1752. return 0;
  1753. }
  1754. #endif /* CONFIG_BLOCK */
  1755. #endif