/drivers/md/dm-thin.c

http://github.com/mirrors/linux · C · 4549 lines · 3149 code · 782 blank · 618 comment · 431 complexity · 8b180e4f08d96d2cd5c069d9d49c4789 MD5 · raw file

Large files are truncated click here to view the full file

  1. /*
  2. * Copyright (C) 2011-2012 Red Hat UK.
  3. *
  4. * This file is released under the GPL.
  5. */
  6. #include "dm-thin-metadata.h"
  7. #include "dm-bio-prison-v1.h"
  8. #include "dm.h"
  9. #include <linux/device-mapper.h>
  10. #include <linux/dm-io.h>
  11. #include <linux/dm-kcopyd.h>
  12. #include <linux/jiffies.h>
  13. #include <linux/log2.h>
  14. #include <linux/list.h>
  15. #include <linux/rculist.h>
  16. #include <linux/init.h>
  17. #include <linux/module.h>
  18. #include <linux/slab.h>
  19. #include <linux/vmalloc.h>
  20. #include <linux/sort.h>
  21. #include <linux/rbtree.h>
  22. #define DM_MSG_PREFIX "thin"
  23. /*
  24. * Tunable constants
  25. */
  26. #define ENDIO_HOOK_POOL_SIZE 1024
  27. #define MAPPING_POOL_SIZE 1024
  28. #define COMMIT_PERIOD HZ
  29. #define NO_SPACE_TIMEOUT_SECS 60
  30. static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
  31. DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
  32. "A percentage of time allocated for copy on write");
  33. /*
  34. * The block size of the device holding pool data must be
  35. * between 64KB and 1GB.
  36. */
  37. #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
  38. #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
  39. /*
  40. * Device id is restricted to 24 bits.
  41. */
  42. #define MAX_DEV_ID ((1 << 24) - 1)
  43. /*
  44. * How do we handle breaking sharing of data blocks?
  45. * =================================================
  46. *
  47. * We use a standard copy-on-write btree to store the mappings for the
  48. * devices (note I'm talking about copy-on-write of the metadata here, not
  49. * the data). When you take an internal snapshot you clone the root node
  50. * of the origin btree. After this there is no concept of an origin or a
  51. * snapshot. They are just two device trees that happen to point to the
  52. * same data blocks.
  53. *
  54. * When we get a write in we decide if it's to a shared data block using
  55. * some timestamp magic. If it is, we have to break sharing.
  56. *
  57. * Let's say we write to a shared block in what was the origin. The
  58. * steps are:
  59. *
  60. * i) plug io further to this physical block. (see bio_prison code).
  61. *
  62. * ii) quiesce any read io to that shared data block. Obviously
  63. * including all devices that share this block. (see dm_deferred_set code)
  64. *
  65. * iii) copy the data block to a newly allocate block. This step can be
  66. * missed out if the io covers the block. (schedule_copy).
  67. *
  68. * iv) insert the new mapping into the origin's btree
  69. * (process_prepared_mapping). This act of inserting breaks some
  70. * sharing of btree nodes between the two devices. Breaking sharing only
  71. * effects the btree of that specific device. Btrees for the other
  72. * devices that share the block never change. The btree for the origin
  73. * device as it was after the last commit is untouched, ie. we're using
  74. * persistent data structures in the functional programming sense.
  75. *
  76. * v) unplug io to this physical block, including the io that triggered
  77. * the breaking of sharing.
  78. *
  79. * Steps (ii) and (iii) occur in parallel.
  80. *
  81. * The metadata _doesn't_ need to be committed before the io continues. We
  82. * get away with this because the io is always written to a _new_ block.
  83. * If there's a crash, then:
  84. *
  85. * - The origin mapping will point to the old origin block (the shared
  86. * one). This will contain the data as it was before the io that triggered
  87. * the breaking of sharing came in.
  88. *
  89. * - The snap mapping still points to the old block. As it would after
  90. * the commit.
  91. *
  92. * The downside of this scheme is the timestamp magic isn't perfect, and
  93. * will continue to think that data block in the snapshot device is shared
  94. * even after the write to the origin has broken sharing. I suspect data
  95. * blocks will typically be shared by many different devices, so we're
  96. * breaking sharing n + 1 times, rather than n, where n is the number of
  97. * devices that reference this data block. At the moment I think the
  98. * benefits far, far outweigh the disadvantages.
  99. */
  100. /*----------------------------------------------------------------*/
  101. /*
  102. * Key building.
  103. */
  104. enum lock_space {
  105. VIRTUAL,
  106. PHYSICAL
  107. };
  108. static void build_key(struct dm_thin_device *td, enum lock_space ls,
  109. dm_block_t b, dm_block_t e, struct dm_cell_key *key)
  110. {
  111. key->virtual = (ls == VIRTUAL);
  112. key->dev = dm_thin_dev_id(td);
  113. key->block_begin = b;
  114. key->block_end = e;
  115. }
  116. static void build_data_key(struct dm_thin_device *td, dm_block_t b,
  117. struct dm_cell_key *key)
  118. {
  119. build_key(td, PHYSICAL, b, b + 1llu, key);
  120. }
  121. static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
  122. struct dm_cell_key *key)
  123. {
  124. build_key(td, VIRTUAL, b, b + 1llu, key);
  125. }
  126. /*----------------------------------------------------------------*/
  127. #define THROTTLE_THRESHOLD (1 * HZ)
  128. struct throttle {
  129. struct rw_semaphore lock;
  130. unsigned long threshold;
  131. bool throttle_applied;
  132. };
  133. static void throttle_init(struct throttle *t)
  134. {
  135. init_rwsem(&t->lock);
  136. t->throttle_applied = false;
  137. }
  138. static void throttle_work_start(struct throttle *t)
  139. {
  140. t->threshold = jiffies + THROTTLE_THRESHOLD;
  141. }
  142. static void throttle_work_update(struct throttle *t)
  143. {
  144. if (!t->throttle_applied && jiffies > t->threshold) {
  145. down_write(&t->lock);
  146. t->throttle_applied = true;
  147. }
  148. }
  149. static void throttle_work_complete(struct throttle *t)
  150. {
  151. if (t->throttle_applied) {
  152. t->throttle_applied = false;
  153. up_write(&t->lock);
  154. }
  155. }
  156. static void throttle_lock(struct throttle *t)
  157. {
  158. down_read(&t->lock);
  159. }
  160. static void throttle_unlock(struct throttle *t)
  161. {
  162. up_read(&t->lock);
  163. }
  164. /*----------------------------------------------------------------*/
  165. /*
  166. * A pool device ties together a metadata device and a data device. It
  167. * also provides the interface for creating and destroying internal
  168. * devices.
  169. */
  170. struct dm_thin_new_mapping;
  171. /*
  172. * The pool runs in various modes. Ordered in degraded order for comparisons.
  173. */
  174. enum pool_mode {
  175. PM_WRITE, /* metadata may be changed */
  176. PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */
  177. /*
  178. * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY.
  179. */
  180. PM_OUT_OF_METADATA_SPACE,
  181. PM_READ_ONLY, /* metadata may not be changed */
  182. PM_FAIL, /* all I/O fails */
  183. };
  184. struct pool_features {
  185. enum pool_mode mode;
  186. bool zero_new_blocks:1;
  187. bool discard_enabled:1;
  188. bool discard_passdown:1;
  189. bool error_if_no_space:1;
  190. };
  191. struct thin_c;
  192. typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
  193. typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
  194. typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
  195. #define CELL_SORT_ARRAY_SIZE 8192
  196. struct pool {
  197. struct list_head list;
  198. struct dm_target *ti; /* Only set if a pool target is bound */
  199. struct mapped_device *pool_md;
  200. struct block_device *data_dev;
  201. struct block_device *md_dev;
  202. struct dm_pool_metadata *pmd;
  203. dm_block_t low_water_blocks;
  204. uint32_t sectors_per_block;
  205. int sectors_per_block_shift;
  206. struct pool_features pf;
  207. bool low_water_triggered:1; /* A dm event has been sent */
  208. bool suspended:1;
  209. bool out_of_data_space:1;
  210. struct dm_bio_prison *prison;
  211. struct dm_kcopyd_client *copier;
  212. struct work_struct worker;
  213. struct workqueue_struct *wq;
  214. struct throttle throttle;
  215. struct delayed_work waker;
  216. struct delayed_work no_space_timeout;
  217. unsigned long last_commit_jiffies;
  218. unsigned ref_count;
  219. spinlock_t lock;
  220. struct bio_list deferred_flush_bios;
  221. struct bio_list deferred_flush_completions;
  222. struct list_head prepared_mappings;
  223. struct list_head prepared_discards;
  224. struct list_head prepared_discards_pt2;
  225. struct list_head active_thins;
  226. struct dm_deferred_set *shared_read_ds;
  227. struct dm_deferred_set *all_io_ds;
  228. struct dm_thin_new_mapping *next_mapping;
  229. process_bio_fn process_bio;
  230. process_bio_fn process_discard;
  231. process_cell_fn process_cell;
  232. process_cell_fn process_discard_cell;
  233. process_mapping_fn process_prepared_mapping;
  234. process_mapping_fn process_prepared_discard;
  235. process_mapping_fn process_prepared_discard_pt2;
  236. struct dm_bio_prison_cell **cell_sort_array;
  237. mempool_t mapping_pool;
  238. struct bio flush_bio;
  239. };
  240. static void metadata_operation_failed(struct pool *pool, const char *op, int r);
  241. static enum pool_mode get_pool_mode(struct pool *pool)
  242. {
  243. return pool->pf.mode;
  244. }
  245. static void notify_of_pool_mode_change(struct pool *pool)
  246. {
  247. const char *descs[] = {
  248. "write",
  249. "out-of-data-space",
  250. "read-only",
  251. "read-only",
  252. "fail"
  253. };
  254. const char *extra_desc = NULL;
  255. enum pool_mode mode = get_pool_mode(pool);
  256. if (mode == PM_OUT_OF_DATA_SPACE) {
  257. if (!pool->pf.error_if_no_space)
  258. extra_desc = " (queue IO)";
  259. else
  260. extra_desc = " (error IO)";
  261. }
  262. dm_table_event(pool->ti->table);
  263. DMINFO("%s: switching pool to %s%s mode",
  264. dm_device_name(pool->pool_md),
  265. descs[(int)mode], extra_desc ? : "");
  266. }
  267. /*
  268. * Target context for a pool.
  269. */
  270. struct pool_c {
  271. struct dm_target *ti;
  272. struct pool *pool;
  273. struct dm_dev *data_dev;
  274. struct dm_dev *metadata_dev;
  275. struct dm_target_callbacks callbacks;
  276. dm_block_t low_water_blocks;
  277. struct pool_features requested_pf; /* Features requested during table load */
  278. struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */
  279. };
  280. /*
  281. * Target context for a thin.
  282. */
  283. struct thin_c {
  284. struct list_head list;
  285. struct dm_dev *pool_dev;
  286. struct dm_dev *origin_dev;
  287. sector_t origin_size;
  288. dm_thin_id dev_id;
  289. struct pool *pool;
  290. struct dm_thin_device *td;
  291. struct mapped_device *thin_md;
  292. bool requeue_mode:1;
  293. spinlock_t lock;
  294. struct list_head deferred_cells;
  295. struct bio_list deferred_bio_list;
  296. struct bio_list retry_on_resume_list;
  297. struct rb_root sort_bio_list; /* sorted list of deferred bios */
  298. /*
  299. * Ensures the thin is not destroyed until the worker has finished
  300. * iterating the active_thins list.
  301. */
  302. refcount_t refcount;
  303. struct completion can_destroy;
  304. };
  305. /*----------------------------------------------------------------*/
  306. static bool block_size_is_power_of_two(struct pool *pool)
  307. {
  308. return pool->sectors_per_block_shift >= 0;
  309. }
  310. static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
  311. {
  312. return block_size_is_power_of_two(pool) ?
  313. (b << pool->sectors_per_block_shift) :
  314. (b * pool->sectors_per_block);
  315. }
  316. /*----------------------------------------------------------------*/
  317. struct discard_op {
  318. struct thin_c *tc;
  319. struct blk_plug plug;
  320. struct bio *parent_bio;
  321. struct bio *bio;
  322. };
  323. static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent)
  324. {
  325. BUG_ON(!parent);
  326. op->tc = tc;
  327. blk_start_plug(&op->plug);
  328. op->parent_bio = parent;
  329. op->bio = NULL;
  330. }
  331. static int issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e)
  332. {
  333. struct thin_c *tc = op->tc;
  334. sector_t s = block_to_sectors(tc->pool, data_b);
  335. sector_t len = block_to_sectors(tc->pool, data_e - data_b);
  336. return __blkdev_issue_discard(tc->pool_dev->bdev, s, len,
  337. GFP_NOWAIT, 0, &op->bio);
  338. }
  339. static void end_discard(struct discard_op *op, int r)
  340. {
  341. if (op->bio) {
  342. /*
  343. * Even if one of the calls to issue_discard failed, we
  344. * need to wait for the chain to complete.
  345. */
  346. bio_chain(op->bio, op->parent_bio);
  347. bio_set_op_attrs(op->bio, REQ_OP_DISCARD, 0);
  348. submit_bio(op->bio);
  349. }
  350. blk_finish_plug(&op->plug);
  351. /*
  352. * Even if r is set, there could be sub discards in flight that we
  353. * need to wait for.
  354. */
  355. if (r && !op->parent_bio->bi_status)
  356. op->parent_bio->bi_status = errno_to_blk_status(r);
  357. bio_endio(op->parent_bio);
  358. }
  359. /*----------------------------------------------------------------*/
  360. /*
  361. * wake_worker() is used when new work is queued and when pool_resume is
  362. * ready to continue deferred IO processing.
  363. */
  364. static void wake_worker(struct pool *pool)
  365. {
  366. queue_work(pool->wq, &pool->worker);
  367. }
  368. /*----------------------------------------------------------------*/
  369. static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
  370. struct dm_bio_prison_cell **cell_result)
  371. {
  372. int r;
  373. struct dm_bio_prison_cell *cell_prealloc;
  374. /*
  375. * Allocate a cell from the prison's mempool.
  376. * This might block but it can't fail.
  377. */
  378. cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
  379. r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
  380. if (r)
  381. /*
  382. * We reused an old cell; we can get rid of
  383. * the new one.
  384. */
  385. dm_bio_prison_free_cell(pool->prison, cell_prealloc);
  386. return r;
  387. }
  388. static void cell_release(struct pool *pool,
  389. struct dm_bio_prison_cell *cell,
  390. struct bio_list *bios)
  391. {
  392. dm_cell_release(pool->prison, cell, bios);
  393. dm_bio_prison_free_cell(pool->prison, cell);
  394. }
  395. static void cell_visit_release(struct pool *pool,
  396. void (*fn)(void *, struct dm_bio_prison_cell *),
  397. void *context,
  398. struct dm_bio_prison_cell *cell)
  399. {
  400. dm_cell_visit_release(pool->prison, fn, context, cell);
  401. dm_bio_prison_free_cell(pool->prison, cell);
  402. }
  403. static void cell_release_no_holder(struct pool *pool,
  404. struct dm_bio_prison_cell *cell,
  405. struct bio_list *bios)
  406. {
  407. dm_cell_release_no_holder(pool->prison, cell, bios);
  408. dm_bio_prison_free_cell(pool->prison, cell);
  409. }
  410. static void cell_error_with_code(struct pool *pool,
  411. struct dm_bio_prison_cell *cell, blk_status_t error_code)
  412. {
  413. dm_cell_error(pool->prison, cell, error_code);
  414. dm_bio_prison_free_cell(pool->prison, cell);
  415. }
  416. static blk_status_t get_pool_io_error_code(struct pool *pool)
  417. {
  418. return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR;
  419. }
  420. static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
  421. {
  422. cell_error_with_code(pool, cell, get_pool_io_error_code(pool));
  423. }
  424. static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
  425. {
  426. cell_error_with_code(pool, cell, 0);
  427. }
  428. static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
  429. {
  430. cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE);
  431. }
  432. /*----------------------------------------------------------------*/
  433. /*
  434. * A global list of pools that uses a struct mapped_device as a key.
  435. */
  436. static struct dm_thin_pool_table {
  437. struct mutex mutex;
  438. struct list_head pools;
  439. } dm_thin_pool_table;
  440. static void pool_table_init(void)
  441. {
  442. mutex_init(&dm_thin_pool_table.mutex);
  443. INIT_LIST_HEAD(&dm_thin_pool_table.pools);
  444. }
  445. static void pool_table_exit(void)
  446. {
  447. mutex_destroy(&dm_thin_pool_table.mutex);
  448. }
  449. static void __pool_table_insert(struct pool *pool)
  450. {
  451. BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
  452. list_add(&pool->list, &dm_thin_pool_table.pools);
  453. }
  454. static void __pool_table_remove(struct pool *pool)
  455. {
  456. BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
  457. list_del(&pool->list);
  458. }
  459. static struct pool *__pool_table_lookup(struct mapped_device *md)
  460. {
  461. struct pool *pool = NULL, *tmp;
  462. BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
  463. list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
  464. if (tmp->pool_md == md) {
  465. pool = tmp;
  466. break;
  467. }
  468. }
  469. return pool;
  470. }
  471. static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
  472. {
  473. struct pool *pool = NULL, *tmp;
  474. BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
  475. list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
  476. if (tmp->md_dev == md_dev) {
  477. pool = tmp;
  478. break;
  479. }
  480. }
  481. return pool;
  482. }
  483. /*----------------------------------------------------------------*/
  484. struct dm_thin_endio_hook {
  485. struct thin_c *tc;
  486. struct dm_deferred_entry *shared_read_entry;
  487. struct dm_deferred_entry *all_io_entry;
  488. struct dm_thin_new_mapping *overwrite_mapping;
  489. struct rb_node rb_node;
  490. struct dm_bio_prison_cell *cell;
  491. };
  492. static void __merge_bio_list(struct bio_list *bios, struct bio_list *master)
  493. {
  494. bio_list_merge(bios, master);
  495. bio_list_init(master);
  496. }
  497. static void error_bio_list(struct bio_list *bios, blk_status_t error)
  498. {
  499. struct bio *bio;
  500. while ((bio = bio_list_pop(bios))) {
  501. bio->bi_status = error;
  502. bio_endio(bio);
  503. }
  504. }
  505. static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master,
  506. blk_status_t error)
  507. {
  508. struct bio_list bios;
  509. bio_list_init(&bios);
  510. spin_lock_irq(&tc->lock);
  511. __merge_bio_list(&bios, master);
  512. spin_unlock_irq(&tc->lock);
  513. error_bio_list(&bios, error);
  514. }
  515. static void requeue_deferred_cells(struct thin_c *tc)
  516. {
  517. struct pool *pool = tc->pool;
  518. struct list_head cells;
  519. struct dm_bio_prison_cell *cell, *tmp;
  520. INIT_LIST_HEAD(&cells);
  521. spin_lock_irq(&tc->lock);
  522. list_splice_init(&tc->deferred_cells, &cells);
  523. spin_unlock_irq(&tc->lock);
  524. list_for_each_entry_safe(cell, tmp, &cells, user_list)
  525. cell_requeue(pool, cell);
  526. }
  527. static void requeue_io(struct thin_c *tc)
  528. {
  529. struct bio_list bios;
  530. bio_list_init(&bios);
  531. spin_lock_irq(&tc->lock);
  532. __merge_bio_list(&bios, &tc->deferred_bio_list);
  533. __merge_bio_list(&bios, &tc->retry_on_resume_list);
  534. spin_unlock_irq(&tc->lock);
  535. error_bio_list(&bios, BLK_STS_DM_REQUEUE);
  536. requeue_deferred_cells(tc);
  537. }
  538. static void error_retry_list_with_code(struct pool *pool, blk_status_t error)
  539. {
  540. struct thin_c *tc;
  541. rcu_read_lock();
  542. list_for_each_entry_rcu(tc, &pool->active_thins, list)
  543. error_thin_bio_list(tc, &tc->retry_on_resume_list, error);
  544. rcu_read_unlock();
  545. }
  546. static void error_retry_list(struct pool *pool)
  547. {
  548. error_retry_list_with_code(pool, get_pool_io_error_code(pool));
  549. }
  550. /*
  551. * This section of code contains the logic for processing a thin device's IO.
  552. * Much of the code depends on pool object resources (lists, workqueues, etc)
  553. * but most is exclusively called from the thin target rather than the thin-pool
  554. * target.
  555. */
  556. static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
  557. {
  558. struct pool *pool = tc->pool;
  559. sector_t block_nr = bio->bi_iter.bi_sector;
  560. if (block_size_is_power_of_two(pool))
  561. block_nr >>= pool->sectors_per_block_shift;
  562. else
  563. (void) sector_div(block_nr, pool->sectors_per_block);
  564. return block_nr;
  565. }
  566. /*
  567. * Returns the _complete_ blocks that this bio covers.
  568. */
  569. static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
  570. dm_block_t *begin, dm_block_t *end)
  571. {
  572. struct pool *pool = tc->pool;
  573. sector_t b = bio->bi_iter.bi_sector;
  574. sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
  575. b += pool->sectors_per_block - 1ull; /* so we round up */
  576. if (block_size_is_power_of_two(pool)) {
  577. b >>= pool->sectors_per_block_shift;
  578. e >>= pool->sectors_per_block_shift;
  579. } else {
  580. (void) sector_div(b, pool->sectors_per_block);
  581. (void) sector_div(e, pool->sectors_per_block);
  582. }
  583. if (e < b)
  584. /* Can happen if the bio is within a single block. */
  585. e = b;
  586. *begin = b;
  587. *end = e;
  588. }
  589. static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
  590. {
  591. struct pool *pool = tc->pool;
  592. sector_t bi_sector = bio->bi_iter.bi_sector;
  593. bio_set_dev(bio, tc->pool_dev->bdev);
  594. if (block_size_is_power_of_two(pool))
  595. bio->bi_iter.bi_sector =
  596. (block << pool->sectors_per_block_shift) |
  597. (bi_sector & (pool->sectors_per_block - 1));
  598. else
  599. bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
  600. sector_div(bi_sector, pool->sectors_per_block);
  601. }
  602. static void remap_to_origin(struct thin_c *tc, struct bio *bio)
  603. {
  604. bio_set_dev(bio, tc->origin_dev->bdev);
  605. }
  606. static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
  607. {
  608. return op_is_flush(bio->bi_opf) &&
  609. dm_thin_changed_this_transaction(tc->td);
  610. }
  611. static void inc_all_io_entry(struct pool *pool, struct bio *bio)
  612. {
  613. struct dm_thin_endio_hook *h;
  614. if (bio_op(bio) == REQ_OP_DISCARD)
  615. return;
  616. h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
  617. h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
  618. }
  619. static void issue(struct thin_c *tc, struct bio *bio)
  620. {
  621. struct pool *pool = tc->pool;
  622. if (!bio_triggers_commit(tc, bio)) {
  623. generic_make_request(bio);
  624. return;
  625. }
  626. /*
  627. * Complete bio with an error if earlier I/O caused changes to
  628. * the metadata that can't be committed e.g, due to I/O errors
  629. * on the metadata device.
  630. */
  631. if (dm_thin_aborted_changes(tc->td)) {
  632. bio_io_error(bio);
  633. return;
  634. }
  635. /*
  636. * Batch together any bios that trigger commits and then issue a
  637. * single commit for them in process_deferred_bios().
  638. */
  639. spin_lock_irq(&pool->lock);
  640. bio_list_add(&pool->deferred_flush_bios, bio);
  641. spin_unlock_irq(&pool->lock);
  642. }
  643. static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
  644. {
  645. remap_to_origin(tc, bio);
  646. issue(tc, bio);
  647. }
  648. static void remap_and_issue(struct thin_c *tc, struct bio *bio,
  649. dm_block_t block)
  650. {
  651. remap(tc, bio, block);
  652. issue(tc, bio);
  653. }
  654. /*----------------------------------------------------------------*/
  655. /*
  656. * Bio endio functions.
  657. */
  658. struct dm_thin_new_mapping {
  659. struct list_head list;
  660. bool pass_discard:1;
  661. bool maybe_shared:1;
  662. /*
  663. * Track quiescing, copying and zeroing preparation actions. When this
  664. * counter hits zero the block is prepared and can be inserted into the
  665. * btree.
  666. */
  667. atomic_t prepare_actions;
  668. blk_status_t status;
  669. struct thin_c *tc;
  670. dm_block_t virt_begin, virt_end;
  671. dm_block_t data_block;
  672. struct dm_bio_prison_cell *cell;
  673. /*
  674. * If the bio covers the whole area of a block then we can avoid
  675. * zeroing or copying. Instead this bio is hooked. The bio will
  676. * still be in the cell, so care has to be taken to avoid issuing
  677. * the bio twice.
  678. */
  679. struct bio *bio;
  680. bio_end_io_t *saved_bi_end_io;
  681. };
  682. static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
  683. {
  684. struct pool *pool = m->tc->pool;
  685. if (atomic_dec_and_test(&m->prepare_actions)) {
  686. list_add_tail(&m->list, &pool->prepared_mappings);
  687. wake_worker(pool);
  688. }
  689. }
  690. static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
  691. {
  692. unsigned long flags;
  693. struct pool *pool = m->tc->pool;
  694. spin_lock_irqsave(&pool->lock, flags);
  695. __complete_mapping_preparation(m);
  696. spin_unlock_irqrestore(&pool->lock, flags);
  697. }
  698. static void copy_complete(int read_err, unsigned long write_err, void *context)
  699. {
  700. struct dm_thin_new_mapping *m = context;
  701. m->status = read_err || write_err ? BLK_STS_IOERR : 0;
  702. complete_mapping_preparation(m);
  703. }
  704. static void overwrite_endio(struct bio *bio)
  705. {
  706. struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
  707. struct dm_thin_new_mapping *m = h->overwrite_mapping;
  708. bio->bi_end_io = m->saved_bi_end_io;
  709. m->status = bio->bi_status;
  710. complete_mapping_preparation(m);
  711. }
  712. /*----------------------------------------------------------------*/
  713. /*
  714. * Workqueue.
  715. */
  716. /*
  717. * Prepared mapping jobs.
  718. */
  719. /*
  720. * This sends the bios in the cell, except the original holder, back
  721. * to the deferred_bios list.
  722. */
  723. static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
  724. {
  725. struct pool *pool = tc->pool;
  726. unsigned long flags;
  727. int has_work;
  728. spin_lock_irqsave(&tc->lock, flags);
  729. cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
  730. has_work = !bio_list_empty(&tc->deferred_bio_list);
  731. spin_unlock_irqrestore(&tc->lock, flags);
  732. if (has_work)
  733. wake_worker(pool);
  734. }
  735. static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
  736. struct remap_info {
  737. struct thin_c *tc;
  738. struct bio_list defer_bios;
  739. struct bio_list issue_bios;
  740. };
  741. static void __inc_remap_and_issue_cell(void *context,
  742. struct dm_bio_prison_cell *cell)
  743. {
  744. struct remap_info *info = context;
  745. struct bio *bio;
  746. while ((bio = bio_list_pop(&cell->bios))) {
  747. if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD)
  748. bio_list_add(&info->defer_bios, bio);
  749. else {
  750. inc_all_io_entry(info->tc->pool, bio);
  751. /*
  752. * We can't issue the bios with the bio prison lock
  753. * held, so we add them to a list to issue on
  754. * return from this function.
  755. */
  756. bio_list_add(&info->issue_bios, bio);
  757. }
  758. }
  759. }
  760. static void inc_remap_and_issue_cell(struct thin_c *tc,
  761. struct dm_bio_prison_cell *cell,
  762. dm_block_t block)
  763. {
  764. struct bio *bio;
  765. struct remap_info info;
  766. info.tc = tc;
  767. bio_list_init(&info.defer_bios);
  768. bio_list_init(&info.issue_bios);
  769. /*
  770. * We have to be careful to inc any bios we're about to issue
  771. * before the cell is released, and avoid a race with new bios
  772. * being added to the cell.
  773. */
  774. cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
  775. &info, cell);
  776. while ((bio = bio_list_pop(&info.defer_bios)))
  777. thin_defer_bio(tc, bio);
  778. while ((bio = bio_list_pop(&info.issue_bios)))
  779. remap_and_issue(info.tc, bio, block);
  780. }
  781. static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
  782. {
  783. cell_error(m->tc->pool, m->cell);
  784. list_del(&m->list);
  785. mempool_free(m, &m->tc->pool->mapping_pool);
  786. }
  787. static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio)
  788. {
  789. struct pool *pool = tc->pool;
  790. /*
  791. * If the bio has the REQ_FUA flag set we must commit the metadata
  792. * before signaling its completion.
  793. */
  794. if (!bio_triggers_commit(tc, bio)) {
  795. bio_endio(bio);
  796. return;
  797. }
  798. /*
  799. * Complete bio with an error if earlier I/O caused changes to the
  800. * metadata that can't be committed, e.g, due to I/O errors on the
  801. * metadata device.
  802. */
  803. if (dm_thin_aborted_changes(tc->td)) {
  804. bio_io_error(bio);
  805. return;
  806. }
  807. /*
  808. * Batch together any bios that trigger commits and then issue a
  809. * single commit for them in process_deferred_bios().
  810. */
  811. spin_lock_irq(&pool->lock);
  812. bio_list_add(&pool->deferred_flush_completions, bio);
  813. spin_unlock_irq(&pool->lock);
  814. }
  815. static void process_prepared_mapping(struct dm_thin_new_mapping *m)
  816. {
  817. struct thin_c *tc = m->tc;
  818. struct pool *pool = tc->pool;
  819. struct bio *bio = m->bio;
  820. int r;
  821. if (m->status) {
  822. cell_error(pool, m->cell);
  823. goto out;
  824. }
  825. /*
  826. * Commit the prepared block into the mapping btree.
  827. * Any I/O for this block arriving after this point will get
  828. * remapped to it directly.
  829. */
  830. r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);
  831. if (r) {
  832. metadata_operation_failed(pool, "dm_thin_insert_block", r);
  833. cell_error(pool, m->cell);
  834. goto out;
  835. }
  836. /*
  837. * Release any bios held while the block was being provisioned.
  838. * If we are processing a write bio that completely covers the block,
  839. * we already processed it so can ignore it now when processing
  840. * the bios in the cell.
  841. */
  842. if (bio) {
  843. inc_remap_and_issue_cell(tc, m->cell, m->data_block);
  844. complete_overwrite_bio(tc, bio);
  845. } else {
  846. inc_all_io_entry(tc->pool, m->cell->holder);
  847. remap_and_issue(tc, m->cell->holder, m->data_block);
  848. inc_remap_and_issue_cell(tc, m->cell, m->data_block);
  849. }
  850. out:
  851. list_del(&m->list);
  852. mempool_free(m, &pool->mapping_pool);
  853. }
  854. /*----------------------------------------------------------------*/
  855. static void free_discard_mapping(struct dm_thin_new_mapping *m)
  856. {
  857. struct thin_c *tc = m->tc;
  858. if (m->cell)
  859. cell_defer_no_holder(tc, m->cell);
  860. mempool_free(m, &tc->pool->mapping_pool);
  861. }
  862. static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
  863. {
  864. bio_io_error(m->bio);
  865. free_discard_mapping(m);
  866. }
  867. static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
  868. {
  869. bio_endio(m->bio);
  870. free_discard_mapping(m);
  871. }
  872. static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
  873. {
  874. int r;
  875. struct thin_c *tc = m->tc;
  876. r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);
  877. if (r) {
  878. metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
  879. bio_io_error(m->bio);
  880. } else
  881. bio_endio(m->bio);
  882. cell_defer_no_holder(tc, m->cell);
  883. mempool_free(m, &tc->pool->mapping_pool);
  884. }
  885. /*----------------------------------------------------------------*/
  886. static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m,
  887. struct bio *discard_parent)
  888. {
  889. /*
  890. * We've already unmapped this range of blocks, but before we
  891. * passdown we have to check that these blocks are now unused.
  892. */
  893. int r = 0;
  894. bool shared = true;
  895. struct thin_c *tc = m->tc;
  896. struct pool *pool = tc->pool;
  897. dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
  898. struct discard_op op;
  899. begin_discard(&op, tc, discard_parent);
  900. while (b != end) {
  901. /* find start of unmapped run */
  902. for (; b < end; b++) {
  903. r = dm_pool_block_is_shared(pool->pmd, b, &shared);
  904. if (r)
  905. goto out;
  906. if (!shared)
  907. break;
  908. }
  909. if (b == end)
  910. break;
  911. /* find end of run */
  912. for (e = b + 1; e != end; e++) {
  913. r = dm_pool_block_is_shared(pool->pmd, e, &shared);
  914. if (r)
  915. goto out;
  916. if (shared)
  917. break;
  918. }
  919. r = issue_discard(&op, b, e);
  920. if (r)
  921. goto out;
  922. b = e;
  923. }
  924. out:
  925. end_discard(&op, r);
  926. }
  927. static void queue_passdown_pt2(struct dm_thin_new_mapping *m)
  928. {
  929. unsigned long flags;
  930. struct pool *pool = m->tc->pool;
  931. spin_lock_irqsave(&pool->lock, flags);
  932. list_add_tail(&m->list, &pool->prepared_discards_pt2);
  933. spin_unlock_irqrestore(&pool->lock, flags);
  934. wake_worker(pool);
  935. }
  936. static void passdown_endio(struct bio *bio)
  937. {
  938. /*
  939. * It doesn't matter if the passdown discard failed, we still want
  940. * to unmap (we ignore err).
  941. */
  942. queue_passdown_pt2(bio->bi_private);
  943. bio_put(bio);
  944. }
  945. static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
  946. {
  947. int r;
  948. struct thin_c *tc = m->tc;
  949. struct pool *pool = tc->pool;
  950. struct bio *discard_parent;
  951. dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin);
  952. /*
  953. * Only this thread allocates blocks, so we can be sure that the
  954. * newly unmapped blocks will not be allocated before the end of
  955. * the function.
  956. */
  957. r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);
  958. if (r) {
  959. metadata_operation_failed(pool, "dm_thin_remove_range", r);
  960. bio_io_error(m->bio);
  961. cell_defer_no_holder(tc, m->cell);
  962. mempool_free(m, &pool->mapping_pool);
  963. return;
  964. }
  965. /*
  966. * Increment the unmapped blocks. This prevents a race between the
  967. * passdown io and reallocation of freed blocks.
  968. */
  969. r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end);
  970. if (r) {
  971. metadata_operation_failed(pool, "dm_pool_inc_data_range", r);
  972. bio_io_error(m->bio);
  973. cell_defer_no_holder(tc, m->cell);
  974. mempool_free(m, &pool->mapping_pool);
  975. return;
  976. }
  977. discard_parent = bio_alloc(GFP_NOIO, 1);
  978. if (!discard_parent) {
  979. DMWARN("%s: unable to allocate top level discard bio for passdown. Skipping passdown.",
  980. dm_device_name(tc->pool->pool_md));
  981. queue_passdown_pt2(m);
  982. } else {
  983. discard_parent->bi_end_io = passdown_endio;
  984. discard_parent->bi_private = m;
  985. if (m->maybe_shared)
  986. passdown_double_checking_shared_status(m, discard_parent);
  987. else {
  988. struct discard_op op;
  989. begin_discard(&op, tc, discard_parent);
  990. r = issue_discard(&op, m->data_block, data_end);
  991. end_discard(&op, r);
  992. }
  993. }
  994. }
  995. static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)
  996. {
  997. int r;
  998. struct thin_c *tc = m->tc;
  999. struct pool *pool = tc->pool;
  1000. /*
  1001. * The passdown has completed, so now we can decrement all those
  1002. * unmapped blocks.
  1003. */
  1004. r = dm_pool_dec_data_range(pool->pmd, m->data_block,
  1005. m->data_block + (m->virt_end - m->virt_begin));
  1006. if (r) {
  1007. metadata_operation_failed(pool, "dm_pool_dec_data_range", r);
  1008. bio_io_error(m->bio);
  1009. } else
  1010. bio_endio(m->bio);
  1011. cell_defer_no_holder(tc, m->cell);
  1012. mempool_free(m, &pool->mapping_pool);
  1013. }
  1014. static void process_prepared(struct pool *pool, struct list_head *head,
  1015. process_mapping_fn *fn)
  1016. {
  1017. struct list_head maps;
  1018. struct dm_thin_new_mapping *m, *tmp;
  1019. INIT_LIST_HEAD(&maps);
  1020. spin_lock_irq(&pool->lock);
  1021. list_splice_init(head, &maps);
  1022. spin_unlock_irq(&pool->lock);
  1023. list_for_each_entry_safe(m, tmp, &maps, list)
  1024. (*fn)(m);
  1025. }
  1026. /*
  1027. * Deferred bio jobs.
  1028. */
  1029. static int io_overlaps_block(struct pool *pool, struct bio *bio)
  1030. {
  1031. return bio->bi_iter.bi_size ==
  1032. (pool->sectors_per_block << SECTOR_SHIFT);
  1033. }
  1034. static int io_overwrites_block(struct pool *pool, struct bio *bio)
  1035. {
  1036. return (bio_data_dir(bio) == WRITE) &&
  1037. io_overlaps_block(pool, bio);
  1038. }
  1039. static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
  1040. bio_end_io_t *fn)
  1041. {
  1042. *save = bio->bi_end_io;
  1043. bio->bi_end_io = fn;
  1044. }
  1045. static int ensure_next_mapping(struct pool *pool)
  1046. {
  1047. if (pool->next_mapping)
  1048. return 0;
  1049. pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC);
  1050. return pool->next_mapping ? 0 : -ENOMEM;
  1051. }
  1052. static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
  1053. {
  1054. struct dm_thin_new_mapping *m = pool->next_mapping;
  1055. BUG_ON(!pool->next_mapping);
  1056. memset(m, 0, sizeof(struct dm_thin_new_mapping));
  1057. INIT_LIST_HEAD(&m->list);
  1058. m->bio = NULL;
  1059. pool->next_mapping = NULL;
  1060. return m;
  1061. }
  1062. static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
  1063. sector_t begin, sector_t end)
  1064. {
  1065. struct dm_io_region to;
  1066. to.bdev = tc->pool_dev->bdev;
  1067. to.sector = begin;
  1068. to.count = end - begin;
  1069. dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
  1070. }
  1071. static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
  1072. dm_block_t data_begin,
  1073. struct dm_thin_new_mapping *m)
  1074. {
  1075. struct pool *pool = tc->pool;
  1076. struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
  1077. h->overwrite_mapping = m;
  1078. m->bio = bio;
  1079. save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
  1080. inc_all_io_entry(pool, bio);
  1081. remap_and_issue(tc, bio, data_begin);
  1082. }
  1083. /*
  1084. * A partial copy also needs to zero the uncopied region.
  1085. */
  1086. static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
  1087. struct dm_dev *origin, dm_block_t data_origin,
  1088. dm_block_t data_dest,
  1089. struct dm_bio_prison_cell *cell, struct bio *bio,
  1090. sector_t len)
  1091. {
  1092. struct pool *pool = tc->pool;
  1093. struct dm_thin_new_mapping *m = get_next_mapping(pool);
  1094. m->tc = tc;
  1095. m->virt_begin = virt_block;
  1096. m->virt_end = virt_block + 1u;
  1097. m->data_block = data_dest;
  1098. m->cell = cell;
  1099. /*
  1100. * quiesce action + copy action + an extra reference held for the
  1101. * duration of this function (we may need to inc later for a
  1102. * partial zero).
  1103. */
  1104. atomic_set(&m->prepare_actions, 3);
  1105. if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
  1106. complete_mapping_preparation(m); /* already quiesced */
  1107. /*
  1108. * IO to pool_dev remaps to the pool target's data_dev.
  1109. *
  1110. * If the whole block of data is being overwritten, we can issue the
  1111. * bio immediately. Otherwise we use kcopyd to clone the data first.
  1112. */
  1113. if (io_overwrites_block(pool, bio))
  1114. remap_and_issue_overwrite(tc, bio, data_dest, m);
  1115. else {
  1116. struct dm_io_region from, to;
  1117. from.bdev = origin->bdev;
  1118. from.sector = data_origin * pool->sectors_per_block;
  1119. from.count = len;
  1120. to.bdev = tc->pool_dev->bdev;
  1121. to.sector = data_dest * pool->sectors_per_block;
  1122. to.count = len;
  1123. dm_kcopyd_copy(pool->copier, &from, 1, &to,
  1124. 0, copy_complete, m);
  1125. /*
  1126. * Do we need to zero a tail region?
  1127. */
  1128. if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
  1129. atomic_inc(&m->prepare_actions);
  1130. ll_zero(tc, m,
  1131. data_dest * pool->sectors_per_block + len,
  1132. (data_dest + 1) * pool->sectors_per_block);
  1133. }
  1134. }
  1135. complete_mapping_preparation(m); /* drop our ref */
  1136. }
  1137. static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
  1138. dm_block_t data_origin, dm_block_t data_dest,
  1139. struct dm_bio_prison_cell *cell, struct bio *bio)
  1140. {
  1141. schedule_copy(tc, virt_block, tc->pool_dev,
  1142. data_origin, data_dest, cell, bio,
  1143. tc->pool->sectors_per_block);
  1144. }
  1145. static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
  1146. dm_block_t data_block, struct dm_bio_prison_cell *cell,
  1147. struct bio *bio)
  1148. {
  1149. struct pool *pool = tc->pool;
  1150. struct dm_thin_new_mapping *m = get_next_mapping(pool);
  1151. atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
  1152. m->tc = tc;
  1153. m->virt_begin = virt_block;
  1154. m->virt_end = virt_block + 1u;
  1155. m->data_block = data_block;
  1156. m->cell = cell;
  1157. /*
  1158. * If the whole block of data is being overwritten or we are not
  1159. * zeroing pre-existing data, we can issue the bio immediately.
  1160. * Otherwise we use kcopyd to zero the data first.
  1161. */
  1162. if (pool->pf.zero_new_blocks) {
  1163. if (io_overwrites_block(pool, bio))
  1164. remap_and_issue_overwrite(tc, bio, data_block, m);
  1165. else
  1166. ll_zero(tc, m, data_block * pool->sectors_per_block,
  1167. (data_block + 1) * pool->sectors_per_block);
  1168. } else
  1169. process_prepared_mapping(m);
  1170. }
  1171. static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
  1172. dm_block_t data_dest,
  1173. struct dm_bio_prison_cell *cell, struct bio *bio)
  1174. {
  1175. struct pool *pool = tc->pool;
  1176. sector_t virt_block_begin = virt_block * pool->sectors_per_block;
  1177. sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
  1178. if (virt_block_end <= tc->origin_size)
  1179. schedule_copy(tc, virt_block, tc->origin_dev,
  1180. virt_block, data_dest, cell, bio,
  1181. pool->sectors_per_block);
  1182. else if (virt_block_begin < tc->origin_size)
  1183. schedule_copy(tc, virt_block, tc->origin_dev,
  1184. virt_block, data_dest, cell, bio,
  1185. tc->origin_size - virt_block_begin);
  1186. else
  1187. schedule_zero(tc, virt_block, data_dest, cell, bio);
  1188. }
  1189. static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
  1190. static void requeue_bios(struct pool *pool);
  1191. static bool is_read_only_pool_mode(enum pool_mode mode)
  1192. {
  1193. return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY);
  1194. }
  1195. static bool is_read_only(struct pool *pool)
  1196. {
  1197. return is_read_only_pool_mode(get_pool_mode(pool));
  1198. }
  1199. static void check_for_metadata_space(struct pool *pool)
  1200. {
  1201. int r;
  1202. const char *ooms_reason = NULL;
  1203. dm_block_t nr_free;
  1204. r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free);
  1205. if (r)
  1206. ooms_reason = "Could not get free metadata blocks";
  1207. else if (!nr_free)
  1208. ooms_reason = "No free metadata blocks";
  1209. if (ooms_reason && !is_read_only(pool)) {
  1210. DMERR("%s", ooms_reason);
  1211. set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE);
  1212. }
  1213. }
  1214. static void check_for_data_space(struct pool *pool)
  1215. {
  1216. int r;
  1217. dm_block_t nr_free;
  1218. if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
  1219. return;
  1220. r = dm_pool_get_free_block_count(pool->pmd, &nr_free);
  1221. if (r)
  1222. return;
  1223. if (nr_free) {
  1224. set_pool_mode(pool, PM_WRITE);
  1225. requeue_bios(pool);
  1226. }
  1227. }
  1228. /*
  1229. * A non-zero return indicates read_only or fail_io mode.
  1230. * Many callers don't care about the return value.
  1231. */
  1232. static int commit(struct pool *pool)
  1233. {
  1234. int r;
  1235. if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE)
  1236. return -EINVAL;
  1237. r = dm_pool_commit_metadata(pool->pmd);
  1238. if (r)
  1239. metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
  1240. else {
  1241. check_for_metadata_space(pool);
  1242. check_for_data_space(pool);
  1243. }
  1244. return r;
  1245. }
  1246. static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
  1247. {
  1248. if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
  1249. DMWARN("%s: reached low water mark for data device: sending event.",
  1250. dm_device_name(pool->pool_md));
  1251. spin_lock_irq(&pool->lock);
  1252. pool->low_water_triggered = true;
  1253. spin_unlock_irq(&pool->lock);
  1254. dm_table_event(pool->ti->table);
  1255. }
  1256. }
  1257. static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
  1258. {
  1259. int r;
  1260. dm_block_t free_blocks;
  1261. struct pool *pool = tc->pool;
  1262. if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
  1263. return -EINVAL;
  1264. r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
  1265. if (r) {
  1266. metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
  1267. return r;
  1268. }
  1269. check_low_water_mark(pool, free_blocks);
  1270. if (!free_blocks) {
  1271. /*
  1272. * Try to commit to see if that will free up some
  1273. * more space.
  1274. */
  1275. r = commit(pool);
  1276. if (r)
  1277. return r;
  1278. r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
  1279. if (r) {
  1280. metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
  1281. return r;
  1282. }
  1283. if (!free_blocks) {
  1284. set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
  1285. return -ENOSPC;
  1286. }
  1287. }
  1288. r = dm_pool_alloc_data_block(pool->pmd, result);
  1289. if (r) {
  1290. if (r == -ENOSPC)
  1291. set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
  1292. else
  1293. metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
  1294. return r;
  1295. }
  1296. r = dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks);
  1297. if (r) {
  1298. metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count", r);
  1299. return r;
  1300. }
  1301. if (!free_blocks) {
  1302. /* Let's commit before we use up the metadata reserve. */
  1303. r = commit(pool);
  1304. if (r)
  1305. return r;
  1306. }
  1307. return 0;
  1308. }
  1309. /*
  1310. * If we have run out of space, queue bios until the device is
  1311. * resumed, presumably after having been reloaded with more space.
  1312. */
  1313. static void retry_on_resume(struct bio *bio)
  1314. {
  1315. struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
  1316. struct thin_c *tc = h->tc;
  1317. spin_lock_irq(&tc->lock);
  1318. bio_list_add(&tc->retry_on_resume_list, bio);
  1319. spin_unlock_irq(&tc->lock);
  1320. }
  1321. static blk_status_t should_error_unserviceable_bio(struct pool *pool)
  1322. {
  1323. enum pool_mode m = get_pool_mode(pool);
  1324. switch (m) {
  1325. case PM_WRITE:
  1326. /* Shouldn't get here */
  1327. DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
  1328. return BLK_STS_IOERR;
  1329. case PM_OUT_OF_DATA_SPACE:
  1330. return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0;
  1331. case PM_OUT_OF_METADATA_SPACE:
  1332. case PM_READ_ONLY:
  1333. case PM_FAIL:
  1334. return BLK_STS_IOERR;
  1335. default:
  1336. /* Shouldn't get here */
  1337. DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
  1338. return BLK_STS_IOERR;
  1339. }
  1340. }
  1341. static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
  1342. {
  1343. blk_status_t error = should_error_unserviceable_bio(pool);
  1344. if (error) {
  1345. bio->bi_status = error;
  1346. bio_endio(bio);
  1347. } else
  1348. retry_on_resume(bio);
  1349. }
  1350. static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
  1351. {
  1352. struct bio *bio;
  1353. struct bio_list bios;
  1354. blk_status_t error;
  1355. error = should_error_unserviceable_bio(pool);
  1356. if (error) {
  1357. cell_error_with_code(pool, cell, error);
  1358. return;
  1359. }
  1360. bio_list_init(&bios);
  1361. cell_release(pool, cell, &bios);
  1362. while ((bio = bio_list_pop(&bios)))
  1363. retry_on_resume(bio);
  1364. }
  1365. static void process_discard_cell_no_passdown(struct thin_c *tc,
  1366. struct dm_bio_prison_cell *virt_cell)
  1367. {
  1368. struct pool *pool = tc->pool;
  1369. struct dm_thin_new_mapping *m = get_next_mapping(pool);
  1370. /*
  1371. * We don't need to lock the data blocks, since there's no
  1372. * passdown. We only lock data blocks for allocation and breaking sharing.
  1373. */
  1374. m->tc = tc;
  1375. m->virt_begin = virt_cell->key.block_begin;
  1376. m->virt_end = virt_cell->key.block_end;
  1377. m->cell = virt_cell;
  1378. m->bio = virt_cell->holder;
  1379. if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
  1380. pool->process_prepared_discard(m);
  1381. }
  1382. static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
  1383. struct bio *bio)
  1384. {
  1385. struct pool *pool = tc->pool;
  1386. int r;
  1387. bool maybe_shared;
  1388. struct dm_cell_key data_key;
  1389. struct dm_bio_prison_cell *data_cell;
  1390. struct dm_thin_new_mapping *m;
  1391. dm_block_t virt_begin, virt_end, data_begin;
  1392. while (begin != end) {
  1393. r = ensure_next_mapping(pool);
  1394. if (r)
  1395. /* we did our best */
  1396. return;
  1397. r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
  1398. &data_begin, &maybe_shared);
  1399. if (r)
  1400. /*
  1401. * Silently fail, letting any mappings we've
  1402. * created complete.
  1403. */
  1404. break;
  1405. build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key);
  1406. if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
  1407. /* contention, we'll give up with this range */
  1408. begin = virt_end;
  1409. continue;
  1410. }
  1411. /*
  1412. * IO may still be going to the destination block. We must
  1413. * quiesce before we can do the removal.
  1414. */
  1415. m = get_next_mapping(pool);
  1416. m->tc = tc;
  1417. m->maybe_shared = maybe_shared;
  1418. m->virt_begin = virt_begin;
  1419. m->virt_end = virt_end;
  1420. m->data_block = data_begin;
  1421. m->cell = data_cell;
  1422. m->bio = bio;
  1423. /*
  1424. * The parent bio must not complete before sub discard bios are
  1425. * chained to it (see end_discard's bio_chain)!
  1426. *
  1427. * This per-mapping bi_remaining increment is paired with
  1428. * the implicit decrement that occurs via bio_endio() in
  1429. * end_discard().
  1430. */
  1431. bio_inc_remaining(bio);
  1432. if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
  1433. pool->process_prepared_discard(m);
  1434. begin = virt_end;
  1435. }
  1436. }
  1437. static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
  1438. {
  1439. struct bio *bio = virt_cell->holder;
  1440. struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
  1441. /*
  1442. * The virt_cell will only get freed once the origin bio completes.
  1443. * This means it will remain locked while all the individual
  1444. * passdown bios are in flight.
  1445. */
  1446. h->cell = virt_cell;
  1447. break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);
  1448. /*
  1449. * We complete the bio now, knowing that the bi_remaining field
  1450. * will prevent completion until the sub range discards have
  1451. * completed.
  1452. */
  1453. bio_endio(bio);
  1454. }
  1455. static void process_discard_bio(struct thin_c *tc, struct bio *bio)
  1456. {
  1457. dm_block_t begin, end;
  1458. struct dm_cell_key virt_key;
  1459. struct dm_bio_prison_cell *virt_cell;
  1460. get_bio_block_range(tc, bio, &begin, &end);
  1461. if (begin == end) {
  1462. /*
  1463. * The discard covers less than a block.
  1464. */
  1465. bio_endio(bio);
  1466. return;
  1467. }
  1468. build_key(tc->td, VIRTUAL, begin, end, &virt_key);
  1469. if (bio_detain(tc->pool, &virt_key, bio, &virt_cell))
  1470. /*
  1471. * Potential starvation issue: We're relying on the
  1472. * fs/application being well behaved, and not trying to
  1473. * send IO to a region at the same time as discarding it.
  1474. * If they do this persistently then it's possible this
  1475. * cell will never be granted.
  1476. */
  1477. return;
  1478. tc->pool->process_discard_cell(tc, virt_cell);
  1479. }
  1480. static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
  1481. struct dm_cell_key *key,
  1482. struct dm_thin_lookup_result *lookup_result,
  1483. struct dm_bio_prison_cell *cell)
  1484. {
  1485. int r;
  1486. dm_block_t data_block;
  1487. struct pool *pool = tc->pool;
  1488. r = alloc_data_block(tc, &data_block);
  1489. switch (r) {
  1490. case 0:
  1491. schedule_internal_copy(tc, block, lookup_result->block,
  1492. data_block, cell, bio);
  1493. break;
  1494. case -ENOSPC:
  1495. retry_bios_on_resume(pool, cell);
  1496. break;
  1497. default:
  1498. DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
  1499. __func__, r);
  1500. cell_error(pool, cell);
  1501. break;
  1502. }
  1503. }
  1504. static void __remap_and_issue_shared_cell(void *context,
  1505. struct dm_bio_prison_cell *cell)
  1506. {
  1507. struct remap_info *info = context;
  1508. struct bio *bio;
  1509. while ((bio = bio_list_pop(&cell->bios))) {
  1510. if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) ||
  1511. bio_op(bio) == REQ_OP_DISCARD)
  1512. bio_list_add(&info->defer_bios, bio);
  1513. else {
  1514. struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
  1515. h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
  1516. inc_all_io_entry(info->tc->pool, bio);
  1517. bio_list_add(&info->issue_bios, bio);
  1518. }
  1519. }
  1520. }
  1521. static void remap_and_issue_shared_cell(struct thin_c *tc,
  1522. struct dm_bio_prison_cell *cell,
  1523. dm_block_t block)
  1524. {
  1525. struct bio *bio;
  1526. struct remap_info info;
  1527. info.tc = tc;
  1528. bio_list_init(&info.defer_bios);
  1529. bio_list_init(&info.issue_bios);
  1530. cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
  1531. &info, cell);
  1532. while ((bio = bio_list_pop(&info.defer_bios)))
  1533. thin_defer_bio(tc, bio);
  1534. while ((bio = bio_list_pop(&info.issue_bios)))
  1535. remap_and_issue(tc, bio, block);
  1536. }
  1537. static void process_shared_bio(struct thin_c *tc, struct bio *bio,
  1538. dm_block_t block,
  1539. struct dm_thin_lookup_result *lookup_result,
  1540. struct dm_bio_prison_cell *virt_cell)
  1541. {
  1542. struct dm_bio_prison_cell *data_cell;
  1543. struct pool *pool = tc->pool;
  1544. struct dm_cell_key key;
  1545. /*
  1546. * If cell is already occupied, then sharing is already in the process
  1547. * of being broken so we have nothing further to do here.
  1548. */
  1549. build_data_key(tc->td, lookup_result->block, &key);
  1550. if (bio_detain(pool, &key, bio, &data_cell)) {
  1551. cell_defer_no_holder(tc, virt_cell);
  1552. return;
  1553. }
  1554. if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
  1555. break_sharing(tc, bio, block, &key, lookup_result, data_cell);
  1556. cell_defer_no_holder(tc, virt_cell);
  1557. } else {
  1558. struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
  1559. h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
  1560. inc_all_io_entry(pool, bio);
  1561. remap_and_issue(tc, bio, lookup_result->block);
  1562. remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
  1563. remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
  1564. }
  1565. }
  1566. static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
  1567. struct dm_bio_prison_cell *cell)
  1568. {
  1569. int r;
  1570. dm_block_t data_block;
  1571. struct pool *pool = tc->pool;
  1572. /*
  1573. * Remap empty bios (flushes) immediately, without provisioning.
  1574. */
  1575. if (!bio->bi_iter.bi_size) {
  1576. inc_all_io_entry(pool, bio);
  1577. cell_defer_no_holder(tc, cell);
  1578. remap_and_issue(tc, bio, 0);
  1579. return;
  1580. }
  1581. /*
  1582. * Fill read bios with zeroes and complete them immediately.
  1583. */
  1584. if (bio_data_dir(bio) == READ) {
  1585. zero_fill_bio(bio);
  1586. cell_defer_no_holder(tc, cell);
  1587. bio_endio(bio);
  1588. return;
  1589. }
  1590. r = alloc_data_block(tc, &data_block);
  1591. switch (r) {
  1592. case 0:
  1593. if (tc->origin_dev)
  1594. schedule_external_copy(tc, block, data_block, cell, bio);
  1595. else
  1596. schedule_zero(tc, block, data_block, cell, bio);
  1597. break;
  1598. case -ENOSPC:
  1599. retry_bios_on_resume(pool, cell);
  1600. break;
  1601. default:
  1602. DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
  1603. __func__, r);
  1604. cell_error(pool, cell);
  1605. break;
  1606. }
  1607. }
  1608. static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
  1609. {
  1610. int r;
  1611. struct pool *pool = tc->pool;
  1612. struct bio *bio = cell->holder;
  1613. dm_block_t block = get_bio_block(tc, bio);
  1614. struct dm_thin_lookup_result lookup_result;
  1615. if (tc->requeue_mode) {
  1616. cell_requeue(pool, cell);
  1617. return;
  1618. }
  1619. r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
  1620. switch (r) {
  1621. case 0:
  1622. if (lookup_result.shared)
  1623. process_shared_bio(tc, bio, block, &lookup_result, cell);
  1624. else {
  1625. inc_all_io_entry(pool, bio);
  1626. remap_and_issue(tc, bio, lookup_result.block);
  1627. inc_remap_and_issue_cell(tc, cell, lookup_result.block);
  1628. }
  1629. break;
  1630. case -ENODATA:
  1631. if (bio_data_dir(bio) == READ && tc->origin_dev) {
  1632. inc_all_io_entry(pool, bio);
  1633. cell_defer_no_holder(tc, cell);
  1634. if (bio_end_sector(bio) <= tc->origin_size)
  1635. remap_to_origin_and_issue(tc, bio);
  1636. else if (bio->bi_iter.bi_sector < tc->origin_size)