Loading drivers/md/dm-bufio.c +9 −0 Original line number Diff line number Diff line Loading @@ -628,6 +628,7 @@ static void bio_complete(struct bio *bio) { struct dm_buffer *b = bio->bi_private; blk_status_t status = bio->bi_status; bio_uninit(bio); kfree(bio); b->end_io(b, status); Loading Loading @@ -660,6 +661,7 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, do { unsigned int this_step = min((unsigned int)(PAGE_SIZE - offset_in_page(ptr)), len); if (!bio_add_page(bio, virt_to_page(ptr), this_step, offset_in_page(ptr))) { bio_put(bio); Loading Loading @@ -782,6 +784,7 @@ static void __write_dirty_buffer(struct dm_buffer *b, static void __flush_write_list(struct list_head *write_list) { struct blk_plug plug; blk_start_plug(&plug); while (!list_empty(write_list)) { struct dm_buffer *b = Loading Loading @@ -1179,6 +1182,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, for (; n_blocks--; block++) { int need_submit; struct dm_buffer *b; b = __bufio_new(c, block, NF_PREFETCH, &need_submit, &write_list); if (unlikely(!list_empty(&write_list))) { Loading Loading @@ -1463,6 +1467,7 @@ void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block) __link_buffer(b, new_block, LIST_DIRTY); } else { sector_t old_block; wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); /* Loading Loading @@ -1553,6 +1558,7 @@ EXPORT_SYMBOL_GPL(dm_bufio_get_block_size); sector_t dm_bufio_get_device_size(struct dm_bufio_client *c) { sector_t s = bdev_nr_sectors(c->bdev); if (s >= c->start) s -= c->start; else Loading Loading @@ -1668,10 +1674,12 @@ static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp) static unsigned long get_retain_buffers(struct dm_bufio_client *c) { unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes); if (likely(c->sectors_per_block_bits >= 0)) retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; else retain_bytes /= c->block_size; return retain_bytes; } Loading Loading @@ -1806,6 +1814,7 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign if (block_size <= KMALLOC_MAX_SIZE && (block_size < PAGE_SIZE || !is_power_of_2(block_size))) { unsigned int align = min(1U << __ffs(block_size), (unsigned int)PAGE_SIZE); snprintf(slab_name, sizeof slab_name, "dm_bufio_cache-%u", block_size); c->slab_cache = kmem_cache_create(slab_name, block_size, align, SLAB_RECLAIM_ACCOUNT, NULL); Loading drivers/md/dm-cache-metadata.c +6 −0 Original line number Diff line number Diff line Loading @@ -535,6 +535,7 @@ static int __create_persistent_data_objects(struct dm_cache_metadata *cmd, bool may_format_device) { int r; cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT, CACHE_MAX_CONCURRENT_LOCKS); if (IS_ERR(cmd->bm)) { Loading Loading @@ -568,6 +569,7 @@ static void update_flags(struct cache_disk_superblock *disk_super, flags_mutator mutator) { uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags)); disk_super->flags = cpu_to_le32(sb_flags); } Loading Loading @@ -732,6 +734,7 @@ static int __commit_transaction(struct dm_cache_metadata *cmd, static __le64 pack_value(dm_oblock_t block, unsigned int flags) { uint64_t value = from_oblock(block); value <<= 16; value = value | (flags & FLAGS_MASK); return cpu_to_le64(value); Loading @@ -741,6 +744,7 @@ static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned int *flag { uint64_t value = le64_to_cpu(value_le); uint64_t b = value >> 16; *block = to_oblock(b); *flags = value & FLAGS_MASK; } Loading Loading @@ -1254,6 +1258,7 @@ static int __insert(struct dm_cache_metadata *cmd, { int r; __le64 value = pack_value(oblock, M_VALID); __dm_bless_for_disk(&value); r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock), Loading Loading @@ -1580,6 +1585,7 @@ static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned int nr_bi { int r; unsigned int i; for (i = 0; i < nr_bits; i++) { r = __dirty(cmd, to_cblock(i), test_bit(i, bits)); if (r) Loading drivers/md/dm-cache-policy-internal.h +3 −0 Original line number Diff line number Diff line Loading @@ -89,6 +89,7 @@ static inline int policy_emit_config_values(struct dm_cache_policy *p, char *res unsigned int maxlen, ssize_t *sz_ptr) { ssize_t sz = *sz_ptr; if (p->emit_config_values) return p->emit_config_values(p, result, maxlen, sz_ptr); Loading Loading @@ -121,12 +122,14 @@ static inline size_t bitset_size_in_bytes(unsigned int nr_entries) static inline unsigned long *alloc_bitset(unsigned int nr_entries) { size_t s = bitset_size_in_bytes(nr_entries); return vzalloc(s); } static inline void clear_bitset(void *bitset, unsigned int nr_entries) { size_t s = bitset_size_in_bytes(nr_entries); memset(bitset, 0, s); } Loading drivers/md/dm-cache-policy-smq.c +2 −0 Original line number Diff line number Diff line Loading @@ -624,6 +624,7 @@ static void __h_insert(struct smq_hash_table *ht, unsigned int bucket, struct en static void h_insert(struct smq_hash_table *ht, struct entry *e) { unsigned int h = hash_64(from_oblock(e->oblock), ht->hash_bits); __h_insert(ht, h, e); } Loading Loading @@ -1633,6 +1634,7 @@ static void smq_tick(struct dm_cache_policy *p, bool can_block) static void smq_allow_migrations(struct dm_cache_policy *p, bool allow) { struct smq_policy *mq = to_smq_policy(p); mq->migrations_allowed = allow; } Loading drivers/md/dm-cache-target.c +11 −1 Original line number Diff line number Diff line Loading @@ -534,6 +534,7 @@ static unsigned int lock_level(struct bio *bio) static struct per_bio_data *get_per_bio_data(struct bio *bio) { struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data)); BUG_ON(!pb); return pb; } Loading Loading @@ -690,6 +691,7 @@ static void clear_discard(struct cache *cache, dm_dblock_t b) static bool is_discarded(struct cache *cache, dm_dblock_t b) { int r; spin_lock_irq(&cache->lock); r = test_bit(from_dblock(b), cache->discard_bitset); spin_unlock_irq(&cache->lock); Loading @@ -700,6 +702,7 @@ static bool is_discarded(struct cache *cache, dm_dblock_t b) static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) { int r; spin_lock_irq(&cache->lock); r = test_bit(from_dblock(oblock_to_dblock(cache, b)), cache->discard_bitset); Loading Loading @@ -814,6 +817,7 @@ static void accounted_request(struct cache *cache, struct bio *bio) static void issue_op(struct bio *bio, void *context) { struct cache *cache = context; accounted_request(cache, bio); } Loading Loading @@ -1074,6 +1078,7 @@ static void quiesce(struct dm_cache_migration *mg, static struct dm_cache_migration *ws_to_mg(struct work_struct *ws) { struct continuation *k = container_of(ws, struct continuation, ws); return container_of(k, struct dm_cache_migration, k); } Loading Loading @@ -1225,6 +1230,7 @@ static void mg_complete(struct dm_cache_migration *mg, bool success) static void mg_success(struct work_struct *ws) { struct dm_cache_migration *mg = ws_to_mg(ws); mg_complete(mg, mg->k.input == 0); } Loading Loading @@ -1363,6 +1369,7 @@ static void mg_copy(struct work_struct *ws) * Fallback to a real full copy after doing some tidying up. */ bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio); BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */ mg->overwrite_bio = NULL; inc_io_migrations(mg->cache); Loading Loading @@ -1465,12 +1472,15 @@ static void invalidate_complete(struct dm_cache_migration *mg, bool success) static void invalidate_completed(struct work_struct *ws) { struct dm_cache_migration *mg = ws_to_mg(ws); invalidate_complete(mg, !mg->k.input); } static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock) { int r = policy_invalidate_mapping(cache->policy, cblock); int r; r = policy_invalidate_mapping(cache->policy, cblock); if (!r) { r = dm_cache_remove_mapping(cache->cmd, cblock); if (r) { Loading Loading
drivers/md/dm-bufio.c +9 −0 Original line number Diff line number Diff line Loading @@ -628,6 +628,7 @@ static void bio_complete(struct bio *bio) { struct dm_buffer *b = bio->bi_private; blk_status_t status = bio->bi_status; bio_uninit(bio); kfree(bio); b->end_io(b, status); Loading Loading @@ -660,6 +661,7 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, do { unsigned int this_step = min((unsigned int)(PAGE_SIZE - offset_in_page(ptr)), len); if (!bio_add_page(bio, virt_to_page(ptr), this_step, offset_in_page(ptr))) { bio_put(bio); Loading Loading @@ -782,6 +784,7 @@ static void __write_dirty_buffer(struct dm_buffer *b, static void __flush_write_list(struct list_head *write_list) { struct blk_plug plug; blk_start_plug(&plug); while (!list_empty(write_list)) { struct dm_buffer *b = Loading Loading @@ -1179,6 +1182,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, for (; n_blocks--; block++) { int need_submit; struct dm_buffer *b; b = __bufio_new(c, block, NF_PREFETCH, &need_submit, &write_list); if (unlikely(!list_empty(&write_list))) { Loading Loading @@ -1463,6 +1467,7 @@ void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block) __link_buffer(b, new_block, LIST_DIRTY); } else { sector_t old_block; wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); /* Loading Loading @@ -1553,6 +1558,7 @@ EXPORT_SYMBOL_GPL(dm_bufio_get_block_size); sector_t dm_bufio_get_device_size(struct dm_bufio_client *c) { sector_t s = bdev_nr_sectors(c->bdev); if (s >= c->start) s -= c->start; else Loading Loading @@ -1668,10 +1674,12 @@ static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp) static unsigned long get_retain_buffers(struct dm_bufio_client *c) { unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes); if (likely(c->sectors_per_block_bits >= 0)) retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; else retain_bytes /= c->block_size; return retain_bytes; } Loading Loading @@ -1806,6 +1814,7 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign if (block_size <= KMALLOC_MAX_SIZE && (block_size < PAGE_SIZE || !is_power_of_2(block_size))) { unsigned int align = min(1U << __ffs(block_size), (unsigned int)PAGE_SIZE); snprintf(slab_name, sizeof slab_name, "dm_bufio_cache-%u", block_size); c->slab_cache = kmem_cache_create(slab_name, block_size, align, SLAB_RECLAIM_ACCOUNT, NULL); Loading
drivers/md/dm-cache-metadata.c +6 −0 Original line number Diff line number Diff line Loading @@ -535,6 +535,7 @@ static int __create_persistent_data_objects(struct dm_cache_metadata *cmd, bool may_format_device) { int r; cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT, CACHE_MAX_CONCURRENT_LOCKS); if (IS_ERR(cmd->bm)) { Loading Loading @@ -568,6 +569,7 @@ static void update_flags(struct cache_disk_superblock *disk_super, flags_mutator mutator) { uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags)); disk_super->flags = cpu_to_le32(sb_flags); } Loading Loading @@ -732,6 +734,7 @@ static int __commit_transaction(struct dm_cache_metadata *cmd, static __le64 pack_value(dm_oblock_t block, unsigned int flags) { uint64_t value = from_oblock(block); value <<= 16; value = value | (flags & FLAGS_MASK); return cpu_to_le64(value); Loading @@ -741,6 +744,7 @@ static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned int *flag { uint64_t value = le64_to_cpu(value_le); uint64_t b = value >> 16; *block = to_oblock(b); *flags = value & FLAGS_MASK; } Loading Loading @@ -1254,6 +1258,7 @@ static int __insert(struct dm_cache_metadata *cmd, { int r; __le64 value = pack_value(oblock, M_VALID); __dm_bless_for_disk(&value); r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock), Loading Loading @@ -1580,6 +1585,7 @@ static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned int nr_bi { int r; unsigned int i; for (i = 0; i < nr_bits; i++) { r = __dirty(cmd, to_cblock(i), test_bit(i, bits)); if (r) Loading
drivers/md/dm-cache-policy-internal.h +3 −0 Original line number Diff line number Diff line Loading @@ -89,6 +89,7 @@ static inline int policy_emit_config_values(struct dm_cache_policy *p, char *res unsigned int maxlen, ssize_t *sz_ptr) { ssize_t sz = *sz_ptr; if (p->emit_config_values) return p->emit_config_values(p, result, maxlen, sz_ptr); Loading Loading @@ -121,12 +122,14 @@ static inline size_t bitset_size_in_bytes(unsigned int nr_entries) static inline unsigned long *alloc_bitset(unsigned int nr_entries) { size_t s = bitset_size_in_bytes(nr_entries); return vzalloc(s); } static inline void clear_bitset(void *bitset, unsigned int nr_entries) { size_t s = bitset_size_in_bytes(nr_entries); memset(bitset, 0, s); } Loading
drivers/md/dm-cache-policy-smq.c +2 −0 Original line number Diff line number Diff line Loading @@ -624,6 +624,7 @@ static void __h_insert(struct smq_hash_table *ht, unsigned int bucket, struct en static void h_insert(struct smq_hash_table *ht, struct entry *e) { unsigned int h = hash_64(from_oblock(e->oblock), ht->hash_bits); __h_insert(ht, h, e); } Loading Loading @@ -1633,6 +1634,7 @@ static void smq_tick(struct dm_cache_policy *p, bool can_block) static void smq_allow_migrations(struct dm_cache_policy *p, bool allow) { struct smq_policy *mq = to_smq_policy(p); mq->migrations_allowed = allow; } Loading
drivers/md/dm-cache-target.c +11 −1 Original line number Diff line number Diff line Loading @@ -534,6 +534,7 @@ static unsigned int lock_level(struct bio *bio) static struct per_bio_data *get_per_bio_data(struct bio *bio) { struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data)); BUG_ON(!pb); return pb; } Loading Loading @@ -690,6 +691,7 @@ static void clear_discard(struct cache *cache, dm_dblock_t b) static bool is_discarded(struct cache *cache, dm_dblock_t b) { int r; spin_lock_irq(&cache->lock); r = test_bit(from_dblock(b), cache->discard_bitset); spin_unlock_irq(&cache->lock); Loading @@ -700,6 +702,7 @@ static bool is_discarded(struct cache *cache, dm_dblock_t b) static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) { int r; spin_lock_irq(&cache->lock); r = test_bit(from_dblock(oblock_to_dblock(cache, b)), cache->discard_bitset); Loading Loading @@ -814,6 +817,7 @@ static void accounted_request(struct cache *cache, struct bio *bio) static void issue_op(struct bio *bio, void *context) { struct cache *cache = context; accounted_request(cache, bio); } Loading Loading @@ -1074,6 +1078,7 @@ static void quiesce(struct dm_cache_migration *mg, static struct dm_cache_migration *ws_to_mg(struct work_struct *ws) { struct continuation *k = container_of(ws, struct continuation, ws); return container_of(k, struct dm_cache_migration, k); } Loading Loading @@ -1225,6 +1230,7 @@ static void mg_complete(struct dm_cache_migration *mg, bool success) static void mg_success(struct work_struct *ws) { struct dm_cache_migration *mg = ws_to_mg(ws); mg_complete(mg, mg->k.input == 0); } Loading Loading @@ -1363,6 +1369,7 @@ static void mg_copy(struct work_struct *ws) * Fallback to a real full copy after doing some tidying up. */ bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio); BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */ mg->overwrite_bio = NULL; inc_io_migrations(mg->cache); Loading Loading @@ -1465,12 +1472,15 @@ static void invalidate_complete(struct dm_cache_migration *mg, bool success) static void invalidate_completed(struct work_struct *ws) { struct dm_cache_migration *mg = ws_to_mg(ws); invalidate_complete(mg, !mg->k.input); } static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock) { int r = policy_invalidate_mapping(cache->policy, cblock); int r; r = policy_invalidate_mapping(cache->policy, cblock); if (!r) { r = dm_cache_remove_mapping(cache->cmd, cblock); if (r) { Loading