diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index 13c0836d7..97b4e46a3 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -317,7 +317,7 @@ static int ocf_acp_populate_handle(ocf_parallelize_t parallelize, { struct ocf_acp_populate_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); ocf_cache_line_t cline, portion; uint64_t begin, end; struct acp_cleaning_policy_meta *acp_meta; @@ -479,7 +479,7 @@ void cleaning_policy_acp_populate(ocf_cache_t cache, bool reconstruct, void cleaning_policy_acp_prepopulate(ocf_cache_t cache, ocf_cleaning_op_end_t cmpl, void *priv) { - ocf_cache_line_t entries = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); ocf_cache_line_t cline; uint32_t step = 0; @@ -503,8 +503,9 @@ static int ocf_acp_update_handle(ocf_parallelize_t parallelize, { struct ocf_acp_update_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->hash_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); ocf_cache_line_t hash, cline, portion; uint64_t begin, end; unsigned lock_idx = shard_id % OCF_NUM_GLOBAL_META_LOCKS; @@ -514,11 +515,11 @@ static int ocf_acp_update_handle(ocf_parallelize_t parallelize, begin = portion*shard_id; end = OCF_MIN(portion*(shard_id + 1), entries); - ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_start_shared_access(&cache->metadata_lock, lock_idx); for (hash = begin; hash < end; hash++) { OCF_COND_RESCHED_DEFAULT(step); - ocf_hb_id_naked_lock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_lock_rd(&cache->metadata_lock, hash); cline = ocf_metadata_get_hash(cache, hash); while (cline != terminator) { @@ -529,9 +530,9 @@ static int ocf_acp_update_handle(ocf_parallelize_t parallelize, cline = ocf_metadata_get_collision_next(cache, cline); } - ocf_hb_id_naked_unlock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_unlock_rd(&cache->metadata_lock, hash); } - ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_end_shared_access(&cache->metadata_lock, lock_idx); return 0; } @@ -637,7 +638,7 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, unsigned lock_idx = ocf_metadata_concurrency_next_idx( cache->cleaner.io_queue); - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, lock_idx, core_id, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, lock_idx, core_id, core_line); ocf_engine_lookup_map_entry(cache, &info, core_id, @@ -650,10 +651,11 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, info.coll_idx); } - ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, lock_idx, core_id, + ocf_hb_cline_prot_unlock_rd(&cache->metadata_lock, lock_idx, core_id, core_line); - return locked ? info.coll_idx : cache->device->collision_table_entries; + return locked ? info.coll_idx : + ocf_metadata_terminator_line(&cache->metadata); } static void _acp_handle_flush_error(struct ocf_cache *cache, @@ -774,7 +776,8 @@ static bool _acp_prepare_flush_data(struct acp_context *acp, ocf_cache_line_t cache_line; cache_line = _acp_trylock_dirty(cache, chunk->core_id, core_line); - if (cache_line == cache->device->collision_table_entries) + if (cache_line == + ocf_metadata_terminator_line(&cache->metadata)) continue; ACP_DEBUG_BEGIN(acp, cache_line); diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index a12f9938c..52498955c 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -16,6 +16,7 @@ #include "../utils/utils_parallelize.h" #include "../utils/utils_realloc.h" #include "../concurrency/ocf_cache_line_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_def_priv.h" #include "cleaning_priv.h" @@ -71,7 +72,8 @@ struct alru_context { static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, ocf_cache_line_t head, ocf_cache_line_t tail) { - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); struct alru_cleaning_policy *part_alru; struct cleaning_policy_meta *meta; struct alru_cleaning_policy_meta *old_head; @@ -105,7 +107,8 @@ static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, static void add_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, ocf_cache_line_t cline) { - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); struct alru_cleaning_policy *part_alru; struct cleaning_policy_meta *meta; struct alru_cleaning_policy_meta *entry; @@ -133,7 +136,8 @@ static void remove_alru_list(struct ocf_cache *cache, int partition_id, unsigned int collision_index) { uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; + uint32_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id] .clean_pol->policy.alru; struct alru_cleaning_policy_meta *alru; @@ -235,7 +239,8 @@ static bool is_on_alru_list(struct ocf_cache *cache, int partition_id, unsigned int collision_index) { uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; + uint32_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id] .clean_pol->policy.alru; struct alru_cleaning_policy_meta *alru; @@ -265,8 +270,8 @@ void cleaning_policy_alru_init_cache_block(struct ocf_cache *cache, alru = &ocf_metadata_get_cleaning_policy(cache, cache_line)->meta.alru; alru->timestamp = 0; - alru->lru_prev = cache->device->collision_table_entries; - alru->lru_next = cache->device->collision_table_entries; + alru->lru_prev = ocf_metadata_terminator_line(&cache->metadata); + alru->lru_next = ocf_metadata_terminator_line(&cache->metadata); } void cleaning_policy_alru_purge_cache_block(struct ocf_cache *cache, @@ -324,7 +329,8 @@ void cleaning_policy_alru_set_hot_cache_line(struct ocf_cache *cache, struct alru_cleaning_policy *part_alru = &cache->user_parts[part_id] .clean_pol->policy.alru; uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; + uint32_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); struct alru_cleaning_policy_meta *alru; ENV_WARN_ON(!metadata_test_dirty(cache, cache_line)); @@ -419,7 +425,7 @@ static void add_alru_head_populate(struct ocf_alru_fill_context *context, struct alru_cleaning_policy_meta *entry; struct alru_cleaning_policy_meta *next; - terminator = ocf_metadata_collision_table_entries(cache); + terminator = ocf_metadata_terminator_line(&cache->metadata); curr_head = context->shard[shard_id].part[part_id].head; meta = ocf_metadata_get_cleaning_policy(cache, cline); @@ -453,8 +459,9 @@ static int ocf_alru_populate_handle(ocf_parallelize_t parallelize, { struct ocf_alru_fill_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->collision_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); unsigned part_size[OCF_USER_IO_CLASS_MAX] = {}; struct ocf_user_part *user_part; struct alru_cleaning_policy *part_alru; @@ -505,8 +512,9 @@ static int ocf_alru_prepopulate_handle(ocf_parallelize_t parallelize, { struct ocf_alru_fill_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->collision_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); unsigned part_size[OCF_USER_IO_CLASS_MAX] = {}; struct ocf_user_part *user_part; struct alru_cleaning_policy *part_alru; @@ -592,8 +600,10 @@ static void cleaning_policy_alru_fill(ocf_cache_t cache, /* ALRU initialization */ part_alru = &user_part->clean_pol->policy.alru; env_atomic_set(&part_alru->size, 0); - part_alru->lru_head = cache->device->collision_table_entries; - part_alru->lru_tail = cache->device->collision_table_entries; + part_alru->lru_head = + ocf_metadata_terminator_line(&cache->metadata); + part_alru->lru_tail = + ocf_metadata_terminator_line(&cache->metadata); cache->device->runtime_meta->cleaning_thread_access = 0; } @@ -635,8 +645,9 @@ static int ocf_alru_update_handle(ocf_parallelize_t parallelize, { struct ocf_alru_update_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->hash_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); ocf_cache_line_t hash, cline, portion; uint32_t begin, end; unsigned lock_idx = shard_id % OCF_NUM_GLOBAL_META_LOCKS; @@ -646,11 +657,11 @@ static int ocf_alru_update_handle(ocf_parallelize_t parallelize, begin = portion*shard_id; end = OCF_MIN((uint64_t)portion*(shard_id + 1), entries); - ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_start_shared_access(&cache->metadata_lock, lock_idx); for (hash = begin; hash < end; hash++) { OCF_COND_RESCHED_DEFAULT(step); - ocf_hb_id_naked_lock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_lock_rd(&cache->metadata_lock, hash); cline = ocf_metadata_get_hash(cache, hash); while (cline != terminator) { @@ -661,9 +672,9 @@ static int ocf_alru_update_handle(ocf_parallelize_t parallelize, cline = ocf_metadata_get_collision_next(cache, cline); } - ocf_hb_id_naked_unlock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_unlock_rd(&cache->metadata_lock, hash); } - ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_end_shared_access(&cache->metadata_lock, lock_idx); return 0; } @@ -947,7 +958,7 @@ static bool more_blocks_to_flush(struct ocf_cache *cache, { struct alru_cleaning_policy_meta *alru; - if (cache_line >= cache->device->collision_table_entries) + if (cache_line >= ocf_metadata_line_count(&cache->metadata)) return false; alru = &ocf_metadata_get_cleaning_policy(cache, @@ -1000,13 +1011,18 @@ static int get_data_to_flush(struct alru_context *ctx) last_access = fctx->dirty_ratio_exceeded ? (uint32_t)(~0UL) : compute_timestamp(config); #if OCF_CLEANING_DEBUG == 1 - if (cache_line != cache->device->collision_table_entries) { - alru = &ocf_metadata_get_cleaning_policy( - cache, cache_line)->meta.alru; - OCF_DEBUG_PARAM(cache, "Last access=%u, " - "timestamp=%u rel=%d", - last_access, alru->timestamp, - alru->timestamp < last_access); + { + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); + + if (cache_line != terminator) { + alru = &ocf_metadata_get_cleaning_policy( + cache, cache_line)->meta.alru; + OCF_DEBUG_PARAM(cache, "Last access=%u, " + "timestamp=%u rel=%d", + last_access, alru->timestamp, + alru->timestamp < last_access); + } } #endif @@ -1063,7 +1079,7 @@ static void alru_clean(struct alru_context *ctx) return; } - if (ocf_metadata_try_start_exclusive_access(&cache->metadata.lock)) { + if (ocf_metadata_try_start_exclusive_access(&cache->metadata_lock)) { alru_clean_complete(fctx, 0); return; } @@ -1082,7 +1098,7 @@ static void alru_clean(struct alru_context *ctx) ocf_cleaner_sort_flush_data(fctx->flush_data, to_clean); ocf_cleaner_do_flush_data_async(cache, fctx->flush_data, to_clean, &fctx->attribs); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return; } @@ -1091,7 +1107,7 @@ static void alru_clean(struct alru_context *ctx) env_ticks_to_secs(env_get_tick_count()); end: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); alru_clean_complete(fctx, 0); } diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index bb8b2274e..3cbf55ab3 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -12,7 +12,7 @@ #include "nop.h" #include "acp.h" #include "../metadata/metadata_superblock.h" -#include "../metadata/metadata_structs.h" +#include "../metadata/metadata_priv.h" #include "../ocf_cache_priv.h" #include "ocf_env_refcnt.h" diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index c802a0a43..3729f13c6 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2022-2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -273,5 +274,5 @@ void ocf_cache_line_concurrency_deinit(struct ocf_alock **self) size_t ocf_cache_line_concurrency_size_of(ocf_cache_t cache) { - return ocf_alock_size(cache->device->collision_table_entries); + return ocf_alock_size(ocf_metadata_line_count(&cache->metadata)); } diff --git a/src/concurrency/ocf_concurrency.c b/src/concurrency/ocf_concurrency.c index 594983afa..547b6cdc8 100644 --- a/src/concurrency/ocf_concurrency.c +++ b/src/concurrency/ocf_concurrency.c @@ -1,18 +1,23 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf_concurrency.h" +#include "ocf_metadata_concurrency.h" #include "../metadata/metadata.h" int ocf_concurrency_init(struct ocf_cache *cache) { int result = 0; + ocf_metadata_concurrency_attached_init(&cache->metadata_lock, cache, + ocf_metadata_hash_entries(&cache->metadata)); + result = ocf_cache_line_concurrency_init( &cache->device->concurrency.cache_line, - ocf_metadata_collision_table_entries(cache), + ocf_metadata_line_count(&cache->metadata), cache); if (result) diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c index f3fdc1f60..da7d6114a 100644 --- a/src/concurrency/ocf_metadata_concurrency.c +++ b/src/concurrency/ocf_metadata_concurrency.c @@ -128,57 +128,12 @@ static inline void hash_lock_unlock(struct ocf_cache *cache, int index, int rw) } } -int ocf_metadata_concurrency_attached_init( +void ocf_metadata_concurrency_attached_init( struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache, - uint32_t hash_table_entries, uint32_t colision_table_pages) + uint32_t hash_table_entries) { - uint32_t i; - int err = 0; - metadata_lock->cache = cache; metadata_lock->num_hash_entries = hash_table_entries; - - if (cache->metadata.is_volatile || colision_table_pages == 0) { - metadata_lock->collision_pages = NULL; - metadata_lock->num_collision_pages = 0; - return 0; - } - - metadata_lock->collision_pages = env_vzalloc(sizeof(env_rwsem) * - colision_table_pages); - if (!metadata_lock->collision_pages) - return -OCF_ERR_NO_MEM; - - for (i = 0; i < colision_table_pages; i++) { - err = env_rwsem_init(&metadata_lock->collision_pages[i]); - if (err) - break; - } - if (err) { - while (i--) - env_rwsem_destroy(&metadata_lock->collision_pages[i]); - env_vfree(metadata_lock->collision_pages); - metadata_lock->collision_pages = NULL; - return err; - } - - metadata_lock->num_collision_pages = colision_table_pages; - - return 0; -} - -void ocf_metadata_concurrency_attached_deinit( - struct ocf_metadata_lock *metadata_lock) -{ - uint32_t i; - - if (metadata_lock->collision_pages) { - for (i = 0; i < metadata_lock->num_collision_pages; i++) - env_rwsem_destroy(&metadata_lock->collision_pages[i]); - env_vfree(metadata_lock->collision_pages); - metadata_lock->collision_pages = NULL; - metadata_lock->num_collision_pages = 0; - } } void ocf_metadata_start_exclusive_access( @@ -418,7 +373,7 @@ void ocf_hb_id_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock, } /* number of hash entries */ -#define _NUM_HASH_ENTRIES req->cache->metadata.lock.num_hash_entries +#define _NUM_HASH_ENTRIES req->cache->metadata_lock.num_hash_entries /* true if hashes are monotonic */ #define _IS_MONOTONIC(req) (req->map[0].hash + req->core_line_count <= \ @@ -493,10 +448,10 @@ void ocf_hb_req_prot_lock_rd(struct ocf_request *req) { ocf_cache_line_t hash; - ocf_metadata_start_shared_access(&req->cache->metadata.lock, + ocf_metadata_start_shared_access(&req->cache->metadata_lock, req->lock_idx); for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_lock(&req->cache->metadata_lock, hash, OCF_METADATA_RD); } } @@ -506,10 +461,10 @@ void ocf_hb_req_prot_unlock_rd(struct ocf_request *req) ocf_cache_line_t hash; for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_unlock(&req->cache->metadata_lock, hash, OCF_METADATA_RD); } - ocf_metadata_end_shared_access(&req->cache->metadata.lock, + ocf_metadata_end_shared_access(&req->cache->metadata_lock, req->lock_idx); } @@ -517,10 +472,10 @@ void ocf_hb_req_prot_lock_wr(struct ocf_request *req) { ocf_cache_line_t hash; - ocf_metadata_start_shared_access(&req->cache->metadata.lock, + ocf_metadata_start_shared_access(&req->cache->metadata_lock, req->lock_idx); for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_lock(&req->cache->metadata_lock, hash, OCF_METADATA_WR); } } @@ -530,11 +485,11 @@ void ocf_hb_req_prot_lock_upgrade(struct ocf_request *req) ocf_cache_line_t hash; for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_unlock(&req->cache->metadata_lock, hash, OCF_METADATA_RD); } for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_lock(&req->cache->metadata_lock, hash, OCF_METADATA_WR); } } @@ -544,37 +499,9 @@ void ocf_hb_req_prot_unlock_wr(struct ocf_request *req) ocf_cache_line_t hash; for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_unlock(&req->cache->metadata_lock, hash, OCF_METADATA_WR); } - ocf_metadata_end_shared_access(&req->cache->metadata.lock, + ocf_metadata_end_shared_access(&req->cache->metadata_lock, req->lock_idx); } - -void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_down_read(&metadata_lock->collision_pages[page]); -} - -void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_up_read(&metadata_lock->collision_pages[page]); -} - -void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_down_write(&metadata_lock->collision_pages[page]); -} - -void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_up_write(&metadata_lock->collision_pages[page]); -} diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h index c4db69bad..71cffcbf3 100644 --- a/src/concurrency/ocf_metadata_concurrency.h +++ b/src/concurrency/ocf_metadata_concurrency.h @@ -23,12 +23,9 @@ int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock); void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock); -int ocf_metadata_concurrency_attached_init( +void ocf_metadata_concurrency_attached_init( struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache, - uint32_t hash_table_entries, uint32_t colision_table_pages); - -void ocf_metadata_concurrency_attached_deinit( - struct ocf_metadata_lock *metadata_lock); + uint32_t hash_table_entries); static inline void ocf_metadata_lru_lock( struct ocf_metadata_lock *metadata_lock, unsigned ev_list) @@ -61,18 +58,18 @@ static inline void ocf_metadata_lru_unlock_all( } #define OCF_METADATA_LRU_LOCK(ev_list) \ - ocf_metadata_lru_lock(&cache->metadata.lock, \ + ocf_metadata_lru_lock(&cache->metadata_lock, \ ev_list) #define OCF_METADATA_LRU_UNLOCK(ev_list) \ - ocf_metadata_lru_unlock(&cache->metadata.lock, \ + ocf_metadata_lru_unlock(&cache->metadata_lock, \ ev_list) #define OCF_METADATA_LRU_LOCK_ALL() \ - ocf_metadata_lru_lock_all(&cache->metadata.lock) + ocf_metadata_lru_lock_all(&cache->metadata_lock) #define OCF_METADATA_LRU_UNLOCK_ALL() \ - ocf_metadata_lru_unlock_all(&cache->metadata.lock) + ocf_metadata_lru_unlock_all(&cache->metadata_lock) static inline void ocf_metadata_partition_lock( struct ocf_metadata_lock *metadata_lock, @@ -154,13 +151,5 @@ void ocf_hb_req_prot_lock_wr(struct ocf_request *req); void ocf_hb_req_prot_unlock_wr(struct ocf_request *req); void ocf_hb_req_prot_lock_upgrade(struct ocf_request *req); -/* collision table page lock interface */ -void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); -void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); -void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); -void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); + #endif /* __OCF_METADATA_CONCURRENCY_H__ */ diff --git a/src/concurrency/ocf_metadata_lock.h b/src/concurrency/ocf_metadata_lock.h new file mode 100644 index 000000000..cabbee9a6 --- /dev/null +++ b/src/concurrency/ocf_metadata_lock.h @@ -0,0 +1,31 @@ +/* + * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2025 Huawei Technologies + * Copyright(c) 2026 Unvertical + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __OCF_METADATA_LOCK_H__ +#define __OCF_METADATA_LOCK_H__ + +#include "ocf/ocf.h" +#include "ocf_env.h" +#include "../metadata/metadata_lru.h" + +#define OCF_METADATA_GLOBAL_LOCK_IDX_BITS 2 +#define OCF_NUM_GLOBAL_META_LOCKS (1 << (OCF_METADATA_GLOBAL_LOCK_IDX_BITS)) + +struct ocf_metadata_global_lock { + env_rwsem sem; +} __attribute__((aligned(64))); + +struct ocf_metadata_lock { + struct ocf_metadata_global_lock global[OCF_NUM_GLOBAL_META_LOCKS]; + /*!< global metadata lock (GML) */ + env_spinlock lru[OCF_NUM_LRU_LISTS]; /*!< Fast locks for lru list */ + env_spinlock partition[OCF_USER_IO_CLASS_MAX]; /* partition lock */ + ocf_cache_t cache; /*!< Parent cache object */ + uint32_t num_hash_entries; /*!< Hash bucket count */ +}; + +#endif /* __OCF_METADATA_LOCK_H__ */ diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index 34d54d4f6..6eb302901 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -15,6 +15,7 @@ #include "cache_engine.h" #include "../ocf_request.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../utils/utils_io.h" #define OCF_ENGINE_DEBUG_IO_NAME "bf" @@ -102,7 +103,7 @@ static int _ocf_backfill_do(struct ocf_request *req) ocf_req_forward_cache_init(req, _ocf_backfill_complete); if (ocf_engine_is_sequential(req)) { - addr = cache->device->metadata_offset; + addr = ocf_metadata_data_offset(&cache->metadata); addr += req->map[0].coll_idx * line_size; addr += req->addr % line_size; @@ -120,7 +121,7 @@ static int _ocf_backfill_do(struct ocf_request *req) } else { addr = req->map[i].coll_idx; addr *= line_size; - addr += cache->device->metadata_offset; + addr += ocf_metadata_data_offset(&cache->metadata); } bytes = line_size; @@ -142,7 +143,7 @@ static int _ocf_backfill_do(struct ocf_request *req) for (; i < (req->core_line_count - 1); i++) { addr_next = req->map[i + 1].coll_idx; addr_next *= line_size; - addr_next += cache->device->metadata_offset; + addr_next += ocf_metadata_data_offset(&cache->metadata); if (addr_next != (addr + bytes)) break; diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index 6868868c1..77803ab2d 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -20,6 +20,7 @@ #include "../ocf_space.h" #include "../promotion/promotion.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" void ocf_engine_error(struct ocf_request *req, bool stop_cache, const char *msg) @@ -50,13 +51,13 @@ void ocf_engine_lookup_map_entry(struct ocf_cache *cache, */ entry->hash = hash; entry->status = LOOKUP_MISS; - entry->coll_idx = cache->device->collision_table_entries; + entry->coll_idx = ocf_metadata_terminator_line(&cache->metadata); entry->core_line = core_line; entry->core_id = core_id; line = ocf_metadata_get_hash(cache, hash); - while (line != cache->device->collision_table_entries) { + while (line != ocf_metadata_terminator_line(&cache->metadata)) { ocf_core_id_t curr_core_id; uint64_t curr_core_line; @@ -82,7 +83,8 @@ static inline int _ocf_engine_check_map_entry(struct ocf_cache *cache, if (entry->status == LOOKUP_MISS) return 0; - ENV_BUG_ON(entry->coll_idx >= cache->device->collision_table_entries); + ENV_BUG_ON(entry->coll_idx >= + ocf_metadata_line_count(&cache->metadata)); ocf_metadata_get_core_info(cache, entry->coll_idx, &_core_id, &_core_line); diff --git a/src/engine/engine_discard.c b/src/engine/engine_discard.c index 20bdcc33d..6c5047608 100644 --- a/src/engine/engine_discard.c +++ b/src/engine/engine_discard.c @@ -14,6 +14,7 @@ #include "../ocf_request.h" #include "../utils/utils_io.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG 0 diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index b35788e5c..5a687ed0f 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -14,6 +14,7 @@ #include "../ocf_request.h" #include "../utils/utils_user_part.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../metadata/metadata.h" #define OCF_ENGINE_DEBUG 0 diff --git a/src/engine/engine_inv.c b/src/engine/engine_inv.c index 7a9ec5ccd..0d70ec77b 100644 --- a/src/engine/engine_inv.c +++ b/src/engine/engine_inv.c @@ -13,6 +13,7 @@ #include "../utils/utils_cache_line.h" #include "../metadata/metadata.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "inv" #include "engine_debug.h" diff --git a/src/engine/engine_io.c b/src/engine/engine_io.c index faaa6af13..78e79af11 100644 --- a/src/engine/engine_io.c +++ b/src/engine/engine_io.c @@ -23,7 +23,7 @@ void ocf_engine_forward_cache_io(struct ocf_request *req, int dir, ocf_req_forward_cache_init(req, callback); - addr = cache->device->metadata_offset; + addr = ocf_metadata_data_offset(&cache->metadata); addr += req->map[first_cl].coll_idx * ocf_line_size(cache); addr += (offset + seek) % ocf_line_size(cache); @@ -44,7 +44,7 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, ocf_req_forward_cache_init(req, callback); if (ocf_engine_is_sequential(req)) { - addr = cache->device->metadata_offset; + addr = ocf_metadata_data_offset(&cache->metadata); addr += req->map[0].coll_idx * ocf_line_size(cache); addr += req->addr % ocf_line_size(cache); @@ -63,7 +63,7 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, } else { addr = req->map[i].coll_idx; addr *= ocf_line_size(cache); - addr += cache->device->metadata_offset; + addr += ocf_metadata_data_offset(&cache->metadata); } bytes = ocf_line_size(cache); @@ -78,7 +78,7 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, for (; i < (req->core_line_count - 1); i++) { addr_next = req->map[i + 1].coll_idx; addr_next *= ocf_line_size(cache); - addr_next += cache->device->metadata_offset; + addr_next += ocf_metadata_data_offset(&cache->metadata); if (addr_next != (addr + bytes)) break; diff --git a/src/engine/engine_prefetch.c b/src/engine/engine_prefetch.c index ceb4bf34c..d6e74a9d7 100644 --- a/src/engine/engine_prefetch.c +++ b/src/engine/engine_prefetch.c @@ -13,6 +13,7 @@ #include "engine_io.h" #include "../ocf_cache_priv.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index f15c05594..0e62fecc0 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -14,6 +14,7 @@ #include "../utils/utils_user_part.h" #include "../metadata/metadata.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "pt" #include "engine_debug.h" diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index ac994fa05..95729f484 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -14,6 +14,7 @@ #include "engine_io.h" #include "cache_engine.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" diff --git a/src/engine/engine_wa.c b/src/engine/engine_wa.c index c78131863..ee1ea5332 100644 --- a/src/engine/engine_wa.c +++ b/src/engine/engine_wa.c @@ -13,6 +13,7 @@ #include "cache_engine.h" #include "../ocf_request.h" #include "../metadata/metadata.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wa" #include "engine_debug.h" diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 1860de467..025e8cfed 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -18,6 +18,7 @@ #include "../utils/utils_request.h" #include "../utils/utils_user_part.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wb" #include "engine_debug.h" diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index ff836fd2a..6ff298f5c 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -11,6 +11,7 @@ #include "engine_common.h" #include "engine_io.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../metadata/metadata.h" diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index 9d771cd2c..a73f9d9c4 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -17,6 +17,7 @@ #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wo" #include "engine_debug.h" @@ -72,7 +73,7 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) s = ocf_map_line_start_block(req, line); e = ocf_map_line_end_block(req, line); - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, req->lock_idx, entry->core_id, entry->core_line); @@ -116,7 +117,7 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) == valid); } - ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_unlock_rd(&cache->metadata_lock, req->lock_idx, entry->core_id, entry->core_line); @@ -136,7 +137,7 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) offset += increment; if (i <= e) { - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, req->lock_idx, entry->core_id, entry->core_line); } diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index 303dc0773..8bd51f3c6 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -16,6 +16,7 @@ #include "../utils/utils_user_part.h" #include "../metadata/metadata.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wt" #include "engine_debug.h" diff --git a/src/engine/engine_zero.c b/src/engine/engine_zero.c index 0a74216f3..c5d241723 100644 --- a/src/engine/engine_zero.c +++ b/src/engine/engine_zero.c @@ -10,6 +10,7 @@ #include "engine_zero.h" #include "engine_common.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_io.h" diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 64d86252d..3cb584df9 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -16,11 +16,10 @@ #include "metadata_io.h" #include "metadata_raw.h" #include "metadata_segment.h" -#include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_cache_line_concurrency.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" #include "../utils/utils_cache_line.h" -#include "../utils/utils_io.h" #include "../utils/utils_pipeline.h" #include "../utils/utils_parallelize.h" @@ -157,8 +156,6 @@ void ocf_metadata_deinit_variable_size(struct ocf_cache *cache) OCF_DEBUG_TRACE(cache); - ocf_metadata_concurrency_attached_deinit(&cache->metadata.lock); - /* * De initialize RAW types */ @@ -167,6 +164,10 @@ void ocf_metadata_deinit_variable_size(struct ocf_cache *cache) ocf_metadata_segment_destroy(cache, ctrl->segment[i]); } ctrl->metadata_layout.pages_variable = 0; + + cache->metadata.line_count = 0; + cache->metadata.hash_entries = 0; + cache->metadata.data_offset = 0; } static inline void ocf_metadata_config_init(ocf_cache_t cache, size_t size) @@ -281,7 +282,7 @@ static int ocf_metadata_init_fixed_size(struct ocf_cache *cache, &ctrl->segment[i], cache, &ctrl->raw_desc[i], - NULL, NULL, + false, superblock); if (result) break; @@ -320,23 +321,6 @@ static int ocf_metadata_init_fixed_size(struct ocf_cache *cache, return 0; } -static void ocf_metadata_flush_lock_collision_page(struct ocf_cache *cache, - struct ocf_metadata_raw *raw, uint32_t page) - -{ - ocf_collision_start_exclusive_access(&cache->metadata.lock, - page); -} - -static void ocf_metadata_flush_unlock_collision_page( - struct ocf_cache *cache, struct ocf_metadata_raw *raw, - uint32_t page) - -{ - ocf_collision_end_exclusive_access(&cache->metadata.lock, - page); -} - /* * Initialize hash metadata interface */ @@ -347,7 +331,6 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, int result = 0; uint32_t i = 0; struct ocf_metadata_ctrl *ctrl = NULL; - ocf_flush_page_synch_t lock_page, unlock_page; uint64_t device_lines; struct ocf_metadata_segment *superblock; @@ -414,20 +397,12 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, if (raw->layout->disabled) continue; - if (i == metadata_segment_collision) { - lock_page = - ocf_metadata_flush_lock_collision_page; - unlock_page = - ocf_metadata_flush_unlock_collision_page; - } else { - lock_page = unlock_page = NULL; - } - + /* Collision segment is flushed asynchronously */ result |= ocf_metadata_segment_init( &ctrl->segment[i], cache, raw, - lock_page, unlock_page, + i == metadata_segment_collision, superblock); if (result) @@ -469,13 +444,12 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, cache->device->runtime_meta = METADATA_MEM_POOL(ctrl, metadata_segment_sb_runtime); - cache->device->collision_table_entries = - ctrl->metadata_layout.cachelines; + cache->metadata.line_count = ctrl->metadata_layout.cachelines; - cache->device->hash_table_entries = + cache->metadata.hash_entries = ctrl->raw_desc[metadata_segment_hash].layout->entries; - cache->device->metadata_offset = + cache->metadata.data_offset = ocf_metadata_layout_pages(&ctrl->metadata_layout) * PAGE_SIZE; @@ -489,19 +463,7 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, line_size / KiB); ocf_cache_log(cache, log_info, "Metadata size on device: %llu kiB\n", - cache->device->metadata_offset / KiB); - - result = ocf_metadata_concurrency_attached_init(&cache->metadata.lock, - cache, ctrl->raw_desc[metadata_segment_hash]. - layout->entries, - (uint32_t)ctrl->raw_desc[metadata_segment_collision]. - layout->pages); - if (result) { - ocf_cache_log(cache, log_err, "Failed to initialize attached " - "metadata concurrency\n"); - ocf_metadata_deinit_variable_size(cache); - return result; - } + ocf_metadata_data_offset(&cache->metadata) / KiB); return 0; } @@ -509,7 +471,8 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, static inline void _ocf_init_collision_entry(struct ocf_cache *cache, ocf_cache_line_t idx) { - ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; + ocf_cache_line_t invalid_idx = + ocf_metadata_terminator_line(&cache->metadata); ocf_metadata_set_collision_info(cache, idx, invalid_idx, invalid_idx); ocf_metadata_set_core_info(cache, idx, @@ -526,7 +489,8 @@ static int ocf_metadata_init_collision_handle(ocf_parallelize_t parallelize, { struct ocf_init_metadata_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t collision_table_entries = cache->device->collision_table_entries; + ocf_cache_line_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); uint32_t entry, portion, begin, end, step=0; portion = OCF_DIV_ROUND_UP((uint64_t)collision_table_entries, shards_cnt); @@ -590,8 +554,10 @@ static int ocf_metadata_init_hash_table_handle(ocf_parallelize_t parallelize, { struct ocf_init_metadata_context *context = priv; ocf_cache_t cache = context->cache; - uint32_t hash_table_entries = cache->device->hash_table_entries; - ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; + uint32_t hash_table_entries = + ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t invalid_idx = + ocf_metadata_terminator_line(&cache->metadata); uint32_t entry, portion, begin, end, step=0; portion = OCF_DIV_ROUND_UP((uint64_t)hash_table_entries, shards_cnt); @@ -1225,7 +1191,7 @@ void ocf_metadata_set_hash(struct ocf_cache *cache, ocf_cache_line_t index, { struct ocf_hash_entry *entry = ocf_metadata_get_hash_ptr(cache, index); - ENV_BUG_ON(line > cache->device->collision_table_entries); + ENV_BUG_ON(line > ocf_metadata_line_count(&cache->metadata)); entry->line = line; } @@ -1280,21 +1246,9 @@ bool ocf_metadata_check(struct ocf_cache *cache, ocf_cache_line_t line) int ocf_metadata_init(struct ocf_cache *cache, ocf_cache_line_size_t cache_line_size) { - int ret; - OCF_DEBUG_TRACE(cache); - ret = ocf_metadata_init_fixed_size(cache, cache_line_size); - if (ret) - return ret; - - ret = ocf_metadata_concurrency_init(&cache->metadata.lock); - if (ret) { - ocf_metadata_deinit_fixed_size(cache); - return ret; - } - - return 0; + return ocf_metadata_init_fixed_size(cache, cache_line_size); } void ocf_metadata_deinit(struct ocf_cache *cache) @@ -1302,7 +1256,6 @@ void ocf_metadata_deinit(struct ocf_cache *cache) OCF_DEBUG_TRACE(cache); ocf_metadata_deinit_fixed_size(cache); - ocf_metadata_concurrency_deinit(&cache->metadata.lock); } void ocf_metadata_error(struct ocf_cache *cache) @@ -1444,7 +1397,7 @@ bool ocf_metadata_is_hit_no_lock(ocf_cache_t cache, ocf_core_id_t core_id, core_line, core_id); ocf_cache_line_t line = ocf_metadata_get_hash(cache, hash); - while (line != cache->device->collision_table_entries) { + while (line != ocf_metadata_terminator_line(&cache->metadata)) { ocf_core_id_t curr_core_id; uint64_t curr_core_line; diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index 3c95933b8..f8939034e 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -9,8 +9,6 @@ #define __METADATA_H__ #include "metadata_common.h" -#include "../ocf_cache_priv.h" -#include "../ocf_ctx_priv.h" #include "metadata_cleaning_policy.h" #include "metadata_lru.h" #include "metadata_partition.h" @@ -214,12 +212,6 @@ typedef void (*ocf_metadata_load_properties_end_t)(void *priv, int error, void ocf_metadata_load_properties(ocf_volume_t volume, ocf_metadata_load_properties_end_t cmpl, void *priv); -static inline ocf_cache_line_t ocf_metadata_collision_table_entries( - struct ocf_cache *cache) -{ - return cache->device->collision_table_entries; -} - void ocf_metadata_zero_superblock(ocf_cache_t cache, ocf_metadata_end_t cmpl, void *context); diff --git a/src/metadata/metadata_collision.c b/src/metadata/metadata_collision.c index 0e4cd1875..e0b5918d7 100644 --- a/src/metadata/metadata_collision.c +++ b/src/metadata/metadata_collision.c @@ -8,7 +8,6 @@ #include "ocf/ocf.h" #include "metadata.h" #include "metadata_internal.h" -#include "../utils/utils_cache_line.h" static inline void ocf_metadata_list_info_set( struct ocf_metadata_list_info *info, @@ -172,9 +171,9 @@ void ocf_metadata_get_collision_info(struct ocf_cache *cache, ocf_metadata_error(cache); if (next) - *next = cache->device->collision_table_entries; + *next = ocf_metadata_terminator_line(&cache->metadata); if (prev) - *prev = cache->device->collision_table_entries; + *prev = ocf_metadata_terminator_line(&cache->metadata); } } @@ -187,8 +186,10 @@ void ocf_metadata_add_to_collision(struct ocf_cache *cache, ocf_cache_line_t hash, ocf_cache_line_t cache_line) { ocf_cache_line_t prev_cache_line = ocf_metadata_get_hash(cache, hash); - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - ocf_cache_line_t hash_entries = cache->device->hash_table_entries; + ocf_cache_line_t line_entries = + ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t hash_entries = + ocf_metadata_hash_entries(&cache->metadata); ENV_BUG_ON(!(hash < hash_entries)); ENV_BUG_ON(!(cache_line < line_entries)); @@ -226,8 +227,10 @@ void ocf_metadata_remove_from_collision(struct ocf_cache *cache, uint64_t core_line; ocf_cache_line_t hash_father; ocf_cache_line_t prev_line, next_line; - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - ocf_cache_line_t hash_entries = cache->device->hash_table_entries; + ocf_cache_line_t line_entries = + ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t hash_entries = + ocf_metadata_hash_entries(&cache->metadata); ENV_BUG_ON(!(line < line_entries)); @@ -269,7 +272,7 @@ void ocf_metadata_start_collision_shared_access(struct ocf_cache *cache, &ctrl->raw_desc[metadata_segment_collision]; uint32_t page = ocf_metadata_raw_page(raw, line); - ocf_collision_start_shared_access(&cache->metadata.lock, page); + ocf_metadata_raw_page_lock_modify(raw, page); } /* must be called under global metadata read(shared) lock */ @@ -282,7 +285,7 @@ void ocf_metadata_end_collision_shared_access(struct ocf_cache *cache, &ctrl->raw_desc[metadata_segment_collision]; uint32_t page = ocf_metadata_raw_page(raw, line); - ocf_collision_end_shared_access(&cache->metadata.lock, page); + ocf_metadata_raw_page_unlock_modify(raw, page); } diff --git a/src/metadata/metadata_core.c b/src/metadata/metadata_core.c index 600b2f712..415690e35 100644 --- a/src/metadata/metadata_core.c +++ b/src/metadata/metadata_core.c @@ -5,7 +5,6 @@ */ #include "ocf/ocf.h" -#include "../ocf_priv.h" #include "metadata.h" #include "metadata_core.h" #include "metadata_internal.h" diff --git a/src/metadata/metadata_io.c b/src/metadata/metadata_io.c index 7145d5c6b..09e002652 100644 --- a/src/metadata/metadata_io.c +++ b/src/metadata/metadata_io.c @@ -7,11 +7,6 @@ #include "metadata.h" #include "metadata_io.h" #include "../ocf_priv.h" -#include "../engine/cache_engine.h" -#include "../engine/engine_common.h" -#include "../engine/engine_bf.h" -#include "../utils/utils_cache_line.h" -#include "../utils/utils_io.h" #include "../ocf_request.h" #include "../ocf_def_priv.h" #include "../concurrency/ocf_mio_concurrency.h" @@ -115,7 +110,7 @@ static int metadata_io_read_i_atomic_step(struct ocf_request *req) ocf_req_forward_cache_init(req, metadata_io_read_i_atomic_step_end); ocf_req_forward_cache_metadata(req, OCF_READ, - cache->device->metadata_offset + + ocf_metadata_data_offset(&cache->metadata) + PAGES_TO_BYTES(context->curr_offset), PAGES_TO_BYTES(context->curr_count), 0); @@ -130,7 +125,7 @@ int metadata_io_read_i_atomic(ocf_cache_t cache, ocf_queue_t queue, void *priv, ocf_metadata_io_end_t compl_hndl) { struct metadata_io_read_i_atomic_context *context; - uint64_t io_sectors_count = cache->device->collision_table_entries * + uint64_t io_sectors_count = ocf_metadata_line_count(&cache->metadata) * ocf_line_blocks(cache); struct ocf_request *req; @@ -207,14 +202,14 @@ static int metadata_io_do(struct ocf_request *req) ctx_data_seek(cache->owner, req->data, ctx_data_seek_begin, 0); - /* Fill with the latest metadata. */ - if (m_req->req.rw == OCF_WRITE) { - ocf_metadata_start_shared_access(&cache->metadata.lock, - m_req->page % OCF_NUM_GLOBAL_META_LOCKS); + /* + * Fill with the latest metadata. Collision segment pages, flushed + * at runtime, are synchronized by the segment page lock. Other + * segments are flushed only from management operations, during which + * their content is not modified concurrently. + */ + if (m_req->req.rw == OCF_WRITE) metadata_io_req_fill(m_req); - ocf_metadata_end_shared_access(&cache->metadata.lock, - m_req->page % OCF_NUM_GLOBAL_META_LOCKS); - } ctx_data_seek(cache->owner, req->data, ctx_data_seek_begin, 0); diff --git a/src/metadata/metadata_misc.c b/src/metadata/metadata_misc.c index e5906acc7..047385498 100644 --- a/src/metadata/metadata_misc.c +++ b/src/metadata/metadata_misc.c @@ -57,7 +57,7 @@ int ocf_metadata_detach_cline_range(ocf_cache_t cache, ocf_cache_line_t begin, if (begin > end) return -OCF_ERR_INVAL; - if (end > cache->device->collision_table_entries) + if (end > ocf_metadata_line_count(&cache->metadata)) return -OCF_ERR_INVAL; for (cline = begin; cline < end; ++cline) { @@ -90,7 +90,7 @@ int ocf_metadata_restore_cline_range(ocf_cache_t cache, ocf_cache_line_t begin, if (begin > end) return -OCF_ERR_INVAL; - if (end > cache->device->collision_table_entries) + if (end > ocf_metadata_line_count(&cache->metadata)) return -OCF_ERR_INVAL; for (cline = begin; cline < end; ++cline) { diff --git a/src/metadata/metadata_misc.h b/src/metadata/metadata_misc.h index 6e1636c1c..acf493e24 100644 --- a/src/metadata/metadata_misc.h +++ b/src/metadata/metadata_misc.h @@ -14,7 +14,8 @@ static inline ocf_cache_line_t ocf_metadata_hash_func(ocf_cache_t cache, uint64_t core_line_num, ocf_core_id_t core_id) { - return ocf_metadata_collision_hash(cache->device->hash_table_entries, + return ocf_metadata_collision_hash( + ocf_metadata_hash_entries(&cache->metadata), core_line_num, core_id); } @@ -23,7 +24,7 @@ static inline ocf_cache_line_t ocf_metadata_hash_next(ocf_cache_t cache, ocf_cache_line_t hash) { return ocf_metadata_collision_hash_next( - cache->device->hash_table_entries, hash); + ocf_metadata_hash_entries(&cache->metadata), hash); } void ocf_metadata_remove_cache_line(struct ocf_cache *cache, diff --git a/src/metadata/metadata_partition.c b/src/metadata/metadata_partition.c index b56d29ccb..252c97667 100644 --- a/src/metadata/metadata_partition.c +++ b/src/metadata/metadata_partition.c @@ -8,7 +8,6 @@ #include "ocf/ocf.h" #include "metadata.h" #include "metadata_internal.h" -#include "../utils/utils_user_part.h" ocf_part_id_t ocf_metadata_get_partition_id(struct ocf_cache *cache, ocf_cache_line_t line) diff --git a/src/metadata/metadata_passive_update.c b/src/metadata/metadata_passive_update.c index 7e6d1ad47..fbc9110ff 100644 --- a/src/metadata/metadata_passive_update.c +++ b/src/metadata/metadata_passive_update.c @@ -15,14 +15,8 @@ #include "metadata_io.h" #include "metadata_raw.h" #include "metadata_segment.h" -#include "../concurrency/ocf_concurrency.h" #include "../ocf_def_priv.h" -#include "../ocf_priv.h" -#include "../utils/utils_cache_line.h" -#include "../utils/utils_io.h" -#include "../utils/utils_pipeline.h" #include "../concurrency/ocf_pio_concurrency.h" -#include "../engine/engine_common.h" #define MAX_PASSIVE_IO_SIZE (32*MiB) diff --git a/src/metadata/metadata_priv.h b/src/metadata/metadata_priv.h new file mode 100644 index 000000000..a177c4f9c --- /dev/null +++ b/src/metadata/metadata_priv.h @@ -0,0 +1,78 @@ +/* + * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2025 Huawei Technologies + * Copyright(c) 2026 Unvertical + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __METADATA_PRIV_H__ +#define __METADATA_PRIV_H__ + +#include "ocf/ocf.h" +#include "ocf_env.h" + +/** + * @file metadata_priv.h + * @brief Metadata private structures + */ + +/** + * @brief Metadata control structure + */ +struct ocf_metadata { + void *priv; + /*!< Private data of metadata service interface */ + + ocf_cache_line_size_t line_size; + /*!< Cache line size */ + + bool is_volatile; + /*!< true if metadata used in volatile mode (RAM only) */ + + ocf_cache_line_t line_count; + /*!< Number of collision table entries */ + + uint32_t hash_entries; + /*!< Number of hash table entries */ + + uint64_t data_offset; + /*!< Start of cache data area on cache device (in bytes) */ +}; + +/** + * @brief Get number of cache lines (collision table entries) + */ +static inline ocf_cache_line_t ocf_metadata_line_count( + const struct ocf_metadata *metadata) +{ + return metadata->line_count; +} + +/** + * @brief Get cache line index used as list terminator / unmapped marker + */ +static inline ocf_cache_line_t ocf_metadata_terminator_line( + const struct ocf_metadata *metadata) +{ + return metadata->line_count; +} + +/** + * @brief Get number of hash table entries + */ +static inline uint32_t ocf_metadata_hash_entries( + const struct ocf_metadata *metadata) +{ + return metadata->hash_entries; +} + +/** + * @brief Get offset of cache data area on cache device (in bytes) + */ +static inline uint64_t ocf_metadata_data_offset( + const struct ocf_metadata *metadata) +{ + return metadata->data_offset; +} + +#endif /* __METADATA_PRIV_H__ */ diff --git a/src/metadata/metadata_raw.c b/src/metadata/metadata_raw.c index db435dec5..916ddf08d 100644 --- a/src/metadata/metadata_raw.c +++ b/src/metadata/metadata_raw.c @@ -77,6 +77,49 @@ static bool _raw_ssd_page_is_valid(struct ocf_metadata_raw *raw, uint32_t page) +static void _raw_page_locks_deinit(struct ocf_metadata_raw *raw) +{ + uint32_t i; + + if (!raw->page_locks) + return; + + for (i = 0; i < raw->layout->pages; i++) + env_rwsem_destroy(&raw->page_locks[i]); + + env_vfree(raw->page_locks); + raw->page_locks = NULL; +} + +static int _raw_page_locks_init(struct ocf_metadata_raw *raw) +{ + uint32_t i; + int result = 0; + + if (!raw->layout->pages) + return 0; + + raw->page_locks = env_vzalloc(sizeof(*raw->page_locks) * + raw->layout->pages); + if (!raw->page_locks) + return -OCF_ERR_NO_MEM; + + for (i = 0; i < raw->layout->pages; i++) { + result = env_rwsem_init(&raw->page_locks[i]); + if (result) + break; + } + + if (result) { + while (i--) + env_rwsem_destroy(&raw->page_locks[i]); + env_vfree(raw->page_locks); + raw->page_locks = NULL; + } + + return result; +} + /* * RAM Implementation - De-Initialize */ @@ -90,49 +133,93 @@ static int _raw_ram_deinit(ocf_cache_t cache, raw->mem_pool = NULL; } + _raw_page_locks_deinit(raw); ocf_mio_concurrency_deinit(&raw->mio_conc); return 0; } /* - * RAM Implementation - Initialize + * RAM Implementation - Initialize common part + * + * @param mio_conc - serialize metadata IOs to the same pages + * @param page_locks - synchronize page copying with content modifications */ -static int _raw_ram_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, - struct ocf_metadata_raw *raw) +static int _raw_ram_init_common(ocf_cache_t cache, + struct ocf_metadata_raw *raw, bool mio_conc, bool page_locks) { size_t mem_pool_size; int ret; OCF_DEBUG_TRACE(cache); - /* TODO: caller should specify explicitly whether to init mio conc? */ - if (lock_page_pfn) { + if (mio_conc) { ret = ocf_mio_concurrency_init(&raw->mio_conc, raw->layout->offset, raw->layout->pages, cache); if (ret) return ret; } + if (page_locks) { + ret = _raw_page_locks_init(raw); + if (ret) { + ocf_mio_concurrency_deinit(&raw->mio_conc); + return ret; + } + } + /* Allocate memory pool for entries */ mem_pool_size = raw->layout->pages; mem_pool_size *= PAGE_SIZE; raw->mem_pool_limit = mem_pool_size; raw->mem_pool = env_secure_alloc(mem_pool_size); if (!raw->mem_pool) { + _raw_page_locks_deinit(raw); ocf_mio_concurrency_deinit(&raw->mio_conc); return -OCF_ERR_NO_MEM; } ENV_BUG_ON(env_memset(raw->mem_pool, mem_pool_size, 0)); - raw->lock_page = lock_page_pfn; - raw->unlock_page = unlock_page_pfn; - return 0; } +/* + * RAM Implementation - Initialize + * + * Asynchronous flush writes pages copied from memory concurrently with + * content modifications and with other flushes, so it requires page locks + * and serialization of IOs to the same pages. + */ +static int _raw_ram_init(ocf_cache_t cache, bool flush_asynch, + struct ocf_metadata_raw *raw) +{ + return _raw_ram_init_common(cache, raw, flush_asynch, flush_asynch); +} + +/* + * RAM VOLATILE Implementation - Initialize + * + * Volatile RAW is never written to the cache device. + */ +static int _raw_volatile_init(ocf_cache_t cache, bool flush_asynch, + struct ocf_metadata_raw *raw) +{ + return _raw_ram_init_common(cache, raw, false, false); +} + +/* + * RAM ATOMIC Implementation - Initialize + * + * Asynchronous flush doesn't write metadata pages, so no IO serialization + * is needed, but flush_all still copies pages concurrently with content + * modifications, which requires page locks. + */ +static int _raw_atomic_init(ocf_cache_t cache, bool flush_asynch, + struct ocf_metadata_raw *raw) +{ + return _raw_ram_init_common(cache, raw, false, flush_asynch); +} + /* * RAM Implementation - Size of */ @@ -364,11 +451,9 @@ static int _raw_ram_flush_all_fill(ocf_cache_t cache, OCF_DEBUG_PARAM(cache, "Line = %u, Page = %u", line, raw_page); - if (raw->lock_page) - raw->lock_page(cache, raw, raw_page); + ocf_metadata_raw_page_lock_copy(raw, raw_page); ctx_data_wr_check(cache->owner, data, _RAW_RAM_ADDR(raw, line), size); - if (raw->unlock_page) - raw->unlock_page(cache, raw, raw_page); + ocf_metadata_raw_page_unlock_copy(raw, raw_page); ctx_data_zero_check(cache->owner, data, PAGE_SIZE - size); @@ -484,11 +569,9 @@ static int _raw_ram_flush_do_asynch_fill(ocf_cache_t cache, OCF_DEBUG_PARAM(cache, "Line = %u, Page = %u", line, raw_page); - if (raw->lock_page) - raw->lock_page(cache, raw, raw_page); + ocf_metadata_raw_page_lock_copy(raw, raw_page); ctx_data_wr_check(cache->owner, data, _RAW_RAM_ADDR(raw, line), size); - if (raw->unlock_page) - raw->unlock_page(cache, raw, raw_page); + ocf_metadata_raw_page_unlock_copy(raw, raw_page); ctx_data_zero_check(cache->owner, data, PAGE_SIZE - size); @@ -671,7 +754,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .flush_do_asynch = raw_dynamic_flush_do_asynch, }, [metadata_raw_type_volatile] = { - .init = _raw_ram_init, + .init = _raw_volatile_init, .deinit = _raw_ram_deinit, .size_of = _raw_ram_size_of, .size_on_ssd = raw_volatile_size_on_ssd, @@ -687,7 +770,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .flush_do_asynch = raw_volatile_flush_do_asynch, }, [metadata_raw_type_atomic] = { - .init = _raw_ram_init, + .init = _raw_atomic_init, .deinit = _raw_ram_deinit, .size_of = _raw_ram_size_of, .size_on_ssd = _raw_ram_size_on_ssd, @@ -708,16 +791,17 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { * RAW Top interface implementation ******************************************************************************/ -int ocf_metadata_raw_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +int ocf_metadata_raw_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw) { ENV_BUG_ON(raw->raw_type < metadata_raw_type_min); ENV_BUG_ON(raw->raw_type >= metadata_raw_type_max); raw->iface = &(IRAW[raw->raw_type]); - return raw->iface->init(cache, lock_page_pfn, unlock_page_pfn, raw); + raw->flush_asynch = flush_asynch; + raw->page_locks = NULL; + + return raw->iface->init(cache, flush_asynch, raw); } int ocf_metadata_raw_deinit(ocf_cache_t cache, diff --git a/src/metadata/metadata_raw.h b/src/metadata/metadata_raw.h index deade2623..cebc4d3cc 100644 --- a/src/metadata/metadata_raw.h +++ b/src/metadata/metadata_raw.h @@ -53,8 +53,6 @@ struct ocf_metadata_raw; /** * @brief Container page lock/unlock callback */ -typedef void (*ocf_flush_page_synch_t)(ocf_cache_t cache, - struct ocf_metadata_raw *raw, uint32_t page); /** * @brief RAW instance descriptor @@ -83,8 +81,9 @@ struct ocf_metadata_raw { void *priv; /*!< Private data - context */ - ocf_flush_page_synch_t lock_page; /*!< Page lock callback */ - ocf_flush_page_synch_t unlock_page; /*!< Page unlock callback */ + bool flush_asynch; /*!< Asynchronous flush enabled */ + + env_rwsem *page_locks; /*!< Page locks (NULL if not used) */ struct ocf_alock *mio_conc; }; @@ -93,9 +92,7 @@ struct ocf_metadata_raw { * RAW container interface */ struct raw_iface { - int (*init)(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + int (*init)(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw); int (*deinit)(ocf_cache_t cache, @@ -150,14 +147,15 @@ struct raw_iface { * @brief Initialize RAW instance * * @param cache - Cache instance - * @param lock_page_pfn - Optional page lock callback - * @param lock_page_pfn - Optional page unlock callback + * @param flush_asynch - Enable asynchronous flush (flush_mark and + * flush_do_asynch). RAW content is then modified at runtime, + * concurrently with any kind of flush, including flush_all, + * and RAW must assure consistency of flushed pages. + * Not supported by all RAW types. * @param raw - RAW descriptor * @return 0 - Operation success, otherwise error */ -int ocf_metadata_raw_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +int ocf_metadata_raw_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw); /** @@ -220,6 +218,62 @@ static inline uint32_t ocf_metadata_raw_page(struct ocf_metadata_raw* raw, return raw->iface->page(raw, entry); } +/** + * @brief Lock RAW page for modification of its content + * + * Multiple entries within the same page can be modified concurrently, + * as modifications of the particular entries are synchronized by + * higher level locks. + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_lock_modify( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_down_read(&raw->page_locks[page]); +} + +/** + * @brief Unlock RAW page locked for modification of its content + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_unlock_modify( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_up_read(&raw->page_locks[page]); +} + +/** + * @brief Lock RAW page for copying consistent snapshot of its content + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_lock_copy( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_down_write(&raw->page_locks[page]); +} + +/** + * @brief Unlock RAW page locked for copying + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_unlock_copy( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_up_write(&raw->page_locks[page]); +} + /** * @brief Access specified element of metadata directly * @@ -320,6 +374,9 @@ static inline void ocf_metadata_raw_flush_mark(ocf_cache_t cache, struct ocf_metadata_raw *raw, struct ocf_request *req, uint32_t map_idx, int to_state, uint8_t start, uint8_t stop) { + if (!raw->flush_asynch) + return; + raw->iface->flush_mark(cache, req, map_idx, to_state, start, stop); } @@ -327,6 +384,11 @@ static inline int ocf_metadata_raw_flush_do_asynch(ocf_cache_t cache, struct ocf_request *req, struct ocf_metadata_raw *raw, ocf_req_end_t complete) { + if (!raw->flush_asynch) { + complete(req, -OCF_ERR_NOT_SUPP); + return -OCF_ERR_NOT_SUPP; + } + return raw->iface->flush_do_asynch(cache, req, raw, complete); } diff --git a/src/metadata/metadata_raw_atomic.c b/src/metadata/metadata_raw_atomic.c index eb7eb8621..257b053c7 100644 --- a/src/metadata/metadata_raw_atomic.c +++ b/src/metadata/metadata_raw_atomic.c @@ -10,7 +10,6 @@ #include "metadata_segment_id.h" #include "metadata_raw.h" #include "metadata_raw_atomic.h" -#include "../utils/utils_io.h" #include "../utils/utils_cache_line.h" #include "../ocf_def_priv.h" @@ -89,7 +88,7 @@ static void _raw_atomic_flush_do_asynch_sec(struct ocf_cache *cache, start_addr = map->coll_idx; start_addr *= ocf_line_size(cache); - start_addr += cache->device->metadata_offset; + start_addr += ocf_metadata_data_offset(&cache->metadata); start_addr += BLOCKS_TO_BYTES(map->start_flush); len = BLOCKS_TO_BYTES(map->stop_flush - map->start_flush); @@ -179,7 +178,7 @@ int raw_atomic_flush_do_asynch(struct ocf_cache *cache, struct ocf_request *req, while (i < clines_to_flush) { start_addr = clines_tab[i]; start_addr *= ocf_line_size(cache); - start_addr += cache->device->metadata_offset; + start_addr += ocf_metadata_data_offset(&cache->metadata); len = ocf_line_size(cache); while (true) { diff --git a/src/metadata/metadata_raw_dynamic.c b/src/metadata/metadata_raw_dynamic.c index c3708ca2a..224b2d554 100644 --- a/src/metadata/metadata_raw_dynamic.c +++ b/src/metadata/metadata_raw_dynamic.c @@ -11,9 +11,6 @@ #include "metadata_raw_dynamic.h" #include "metadata_layout.h" #include "metadata_io.h" -#include "../engine/cache_engine.h" -#include "../engine/engine_common.h" -#include "../utils/utils_io.h" #include "../ocf_request.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" @@ -141,8 +138,6 @@ int raw_dynamic_deinit(ocf_cache_t cache, OCF_DEBUG_TRACE(cache); - ocf_mio_concurrency_deinit(&raw->mio_conc); - for (i = 0; i < raw->layout->pages; i++) env_secure_free(ctrl->pages[i], PAGE_SIZE); @@ -157,47 +152,35 @@ int raw_dynamic_deinit(ocf_cache_t cache, /* * RAM DYNAMIC Implementation - Initialize */ -int raw_dynamic_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +int raw_dynamic_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw) { struct _raw_ctrl *ctrl; size_t size = sizeof(*ctrl) + (sizeof(ctrl->pages[0]) * raw->layout->pages); - int ret; OCF_DEBUG_TRACE(cache); + /* Asynchronous flush is not supported */ + if (flush_asynch) + return -OCF_ERR_NOT_SUPP; + if (raw->layout->entry_size > PAGE_SIZE) return -1; - /* TODO: caller should specify explicitly whether to init mio conc? */ - if (lock_page_pfn) { - ret = ocf_mio_concurrency_init(&raw->mio_conc, - raw->layout->offset, raw->layout->pages, cache); - if (ret) - return ret; - } ctrl = env_vmalloc(size); - if (!ctrl) { - ocf_mio_concurrency_deinit(&raw->mio_conc); + if (!ctrl) return -1; - } ENV_BUG_ON(env_memset(ctrl, size, 0)); if (env_mutex_init(&ctrl->lock)) { - ocf_mio_concurrency_deinit(&raw->mio_conc); env_vfree(ctrl); return -1; } raw->priv = ctrl; - raw->lock_page = lock_page_pfn; - raw->unlock_page = unlock_page_pfn; - return 0; } @@ -532,12 +515,8 @@ static int raw_dynamic_flush_all_fill(ocf_cache_t cache, raw_page = page - context->ssd_pages_offset; OCF_DEBUG_PARAM(cache, "Page = %u", raw_page); - if (raw->lock_page) - raw->lock_page(cache, raw, raw_page); ctx_data_wr_check(cache->owner, data, _raw_dynamic_page_rd(raw, raw_page), PAGE_SIZE); - if (raw->unlock_page) - raw->unlock_page(cache, raw, raw_page); return 0; } @@ -575,7 +554,7 @@ void raw_dynamic_flush_all(ocf_cache_t cache, struct ocf_metadata_raw *raw, context->ssd_pages_offset, raw->layout->pages, 0, raw_dynamic_flush_all_fill, raw_dynamic_flush_all_complete, - raw->mio_conc); + NULL); if (result) OCF_CMPL_RET(priv, result); } diff --git a/src/metadata/metadata_raw_dynamic.h b/src/metadata/metadata_raw_dynamic.h index 79a13fbd7..790bb8c57 100644 --- a/src/metadata/metadata_raw_dynamic.h +++ b/src/metadata/metadata_raw_dynamic.h @@ -15,9 +15,7 @@ /* * RAW DYNAMIC - Initialize */ -int raw_dynamic_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +int raw_dynamic_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw); /* diff --git a/src/metadata/metadata_segment.c b/src/metadata/metadata_segment.c index be66d693a..ef2cc49ba 100644 --- a/src/metadata/metadata_segment.c +++ b/src/metadata/metadata_segment.c @@ -11,13 +11,12 @@ int ocf_metadata_segment_init_in_place( struct ocf_metadata_segment *segment, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock) { int result; - result = ocf_metadata_raw_init(cache, lock_page_pfn, unlock_page_pfn, raw); + result = ocf_metadata_raw_init(cache, flush_asynch, raw); if (result) return result; @@ -32,8 +31,7 @@ int ocf_metadata_segment_init( struct ocf_metadata_segment **self, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock) { struct ocf_metadata_segment *segment; @@ -44,8 +42,7 @@ int ocf_metadata_segment_init( return -OCF_ERR_NO_MEM; result = ocf_metadata_segment_init_in_place(segment, - cache, raw, lock_page_pfn, unlock_page_pfn, - superblock); + cache, raw, flush_asynch, superblock); if (result) env_vfree(segment); diff --git a/src/metadata/metadata_segment.h b/src/metadata/metadata_segment.h index 84b69c3f1..915dca14f 100644 --- a/src/metadata/metadata_segment.h +++ b/src/metadata/metadata_segment.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2020-2021 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -21,16 +22,14 @@ int ocf_metadata_segment_init_in_place( struct ocf_metadata_segment *segment, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock); int ocf_metadata_segment_init( struct ocf_metadata_segment **self, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock); void ocf_metadata_segment_destroy(struct ocf_cache *cache, diff --git a/src/metadata/metadata_status.h b/src/metadata/metadata_status.h index 271552618..169d0ebdd 100644 --- a/src/metadata/metadata_status.h +++ b/src/metadata/metadata_status.h @@ -8,7 +8,7 @@ #ifndef __METADATA_STATUS_H__ #define __METADATA_STATUS_H__ -#include "../concurrency/ocf_metadata_concurrency.h" +#include "../ocf_cache_priv.h" #include "metadata_cache_line.h" /******************************************************************************* diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h deleted file mode 100644 index 3d2693d63..000000000 --- a/src/metadata/metadata_structs.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2025 Huawei Technologies - * Copyright(c) 2026 Unvertical - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef __METADATA_STRUCTS_H__ -#define __METADATA_STRUCTS_H__ - -#include "metadata_common.h" -#include "../ocf_space.h" -#include "../cleaning/cleaning.h" -#include "../ocf_request.h" -#include "metadata_superblock.h" - - -/** - * @file metadata_priv.h - * @brief Metadata private structures - */ - -#define OCF_METADATA_GLOBAL_LOCK_IDX_BITS 2 -#define OCF_NUM_GLOBAL_META_LOCKS (1 << (OCF_METADATA_GLOBAL_LOCK_IDX_BITS)) - -struct ocf_metadata_global_lock { - env_rwsem sem; -} __attribute__((aligned(64))); - -struct ocf_metadata_lock -{ - struct ocf_metadata_global_lock global[OCF_NUM_GLOBAL_META_LOCKS]; - /*!< global metadata lock (GML) */ - env_spinlock lru[OCF_NUM_LRU_LISTS]; /*!< Fast locks for lru list */ - env_spinlock partition[OCF_USER_IO_CLASS_MAX]; /* partition lock */ - env_rwsem *collision_pages; /*!< Collision table page locks */ - ocf_cache_t cache; /*!< Parent cache object */ - uint32_t num_hash_entries; /*!< Hash bucket count */ - uint32_t num_collision_pages; /*!< Collision table page count */ -}; - -/** - * @brief Metadata control structure - */ -struct ocf_metadata { - void *priv; - /*!< Private data of metadata service interface */ - - ocf_cache_line_size_t line_size; - /*!< Cache line size */ - - bool is_volatile; - /*!< true if metadata used in volatile mode (RAM only) */ - - struct ocf_metadata_lock lock; -}; - -#endif /* __METADATA_STRUCTS_H__ */ diff --git a/src/metadata/metadata_superblock.c b/src/metadata/metadata_superblock.c index 58b4ddbfb..ddfda7561 100644 --- a/src/metadata/metadata_superblock.c +++ b/src/metadata/metadata_superblock.c @@ -638,7 +638,7 @@ int ocf_metadata_superblock_init( return -OCF_ERR_NO_MEM; result = ocf_metadata_segment_init_in_place(&sb->segment, cache, - raw, NULL, NULL, &sb->segment); + raw, false, &sb->segment); if (result) { env_vfree(sb); diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index ffdc396c0..95585c2c6 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -601,10 +601,10 @@ static void ocf_mngt_cline_rebuild_metadata(ocf_cache_t cache, hash_index = ocf_metadata_hash_func(cache, core_line, core_id); - ocf_hb_id_naked_lock_wr(&cache->metadata.lock, hash_index); + ocf_hb_id_naked_lock_wr(&cache->metadata_lock, hash_index); ocf_metadata_add_to_collision(cache, core_id, core_line, hash_index, cline); - ocf_hb_id_naked_unlock_wr(&cache->metadata.lock, hash_index); + ocf_hb_id_naked_unlock_wr(&cache->metadata_lock, hash_index); ocf_lru_init_cline(cache, cline); @@ -621,7 +621,7 @@ static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize, ocf_core_id_t core_id; uint64_t core_line; unsigned char step = 0; - const uint64_t entries = ocf_metadata_collision_table_entries(cache); + const uint64_t entries = ocf_metadata_line_count(&cache->metadata); begin = shard_id * OCF_LRU_CHUNK_SIZE; @@ -957,6 +957,14 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, goto mutex_err; } + result = ocf_metadata_concurrency_init(&cache->metadata_lock); + if (result) { + ocf_log(params->ctx, log_err, + "Failed to allocate cache %s metadata lock\n", + new_cache_name); + goto metadata_lock_err; + } + result = !env_refcnt_inc(&cache->refcnt.cache); if (result) { ocf_log(params->ctx, log_crit, @@ -979,6 +987,8 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, return 0; cache_refcnt_inc_err: + ocf_metadata_concurrency_deinit(&cache->metadata_lock); +metadata_lock_err: env_spinlock_destroy(&cache->io_queues_lock); mutex_err: env_mutex_destroy(&cache->flush_mutex); @@ -1570,6 +1580,8 @@ static void _ocf_mngt_init_handle_error(ocf_ctx_t ctx, if (!params->flags.cache_alloc) return; + ocf_metadata_concurrency_deinit(&cache->metadata_lock); + env_spinlock_destroy(&cache->io_queues_lock); env_mutex_destroy(&cache->flush_mutex); @@ -1961,7 +1973,7 @@ static void _ocf_mngt_attach_discard(ocf_pipeline_t pipeline, { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - uint64_t addr = cache->device->metadata_offset; + uint64_t addr = ocf_metadata_data_offset(&cache->metadata); uint64_t length = ocf_volume_get_length(&cache->device->volume) - addr; bool discard = cache->device->volume.features.discard_zeroes; @@ -2381,6 +2393,7 @@ static void _ocf_mngt_cache_dealloc(void *priv) ctx = cache->owner; ocf_metadata_deinit(cache); + ocf_metadata_concurrency_deinit(&cache->metadata_lock); if (!cache->metadata.is_volatile) ocf_metadata_io_close(ctx); @@ -3705,11 +3718,11 @@ int ocf_mngt_cache_promotion_set_policy(ocf_cache_t cache, ocf_promotion_t type) if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_promotion_set_policy(cache->promotion_policy, type); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return result; } @@ -3721,11 +3734,11 @@ int ocf_mngt_cache_promotion_get_policy(ocf_cache_t cache, ocf_promotion_t *type if (ocf_cache_is_standby(cache)) return -OCF_ERR_CACHE_STANDBY; - ocf_metadata_start_shared_access(&cache->metadata.lock, 0); + ocf_metadata_start_shared_access(&cache->metadata_lock, 0); *type = cache->conf_meta->promotion_policy_type; - ocf_metadata_end_shared_access(&cache->metadata.lock, 0); + ocf_metadata_end_shared_access(&cache->metadata_lock, 0); return 0; } @@ -3738,11 +3751,11 @@ int ocf_mngt_cache_promotion_get_param(ocf_cache_t cache, ocf_promotion_t type, if (ocf_cache_is_standby(cache)) return -OCF_ERR_CACHE_STANDBY; - ocf_metadata_start_shared_access(&cache->metadata.lock, 0); + ocf_metadata_start_shared_access(&cache->metadata_lock, 0); result = ocf_promotion_get_param(cache, type, param_id, param_value); - ocf_metadata_end_shared_access(&cache->metadata.lock, 0); + ocf_metadata_end_shared_access(&cache->metadata_lock, 0); return result; } @@ -3758,11 +3771,11 @@ int ocf_mngt_cache_promotion_set_param(ocf_cache_t cache, ocf_promotion_t type, if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_promotion_set_param(cache, type, param_id, param_value); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return result; } @@ -3781,7 +3794,7 @@ int ocf_mngt_cache_prefetch_set_policy(ocf_cache_t cache, ocf_pf_mask_t mask) if (mask & ~valid_mask) return -OCF_ERR_INVAL; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); old_mask = cache->conf_meta->prefetch_mask; cache->conf_meta->prefetch_mask = mask; @@ -3802,7 +3815,7 @@ int ocf_mngt_cache_prefetch_set_policy(ocf_cache_t cache, ocf_pf_mask_t mask) } } - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return 0; } @@ -3814,11 +3827,11 @@ int ocf_mngt_cache_prefetch_get_policy(ocf_cache_t cache, ocf_pf_mask_t *mask) if (ocf_cache_is_standby(cache)) return -OCF_ERR_CACHE_STANDBY; - ocf_metadata_start_shared_access(&cache->metadata.lock, 0); + ocf_metadata_start_shared_access(&cache->metadata_lock, 0); *mask = cache->conf_meta->prefetch_mask; - ocf_metadata_end_shared_access(&cache->metadata.lock, 0); + ocf_metadata_end_shared_access(&cache->metadata_lock, 0); return 0; } @@ -3839,11 +3852,11 @@ int ocf_mngt_cache_prefetch_set_param(ocf_cache_t cache, ocf_pf_id_t pf_id, if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); ret = ocf_prefetch_set_param(cache, pf_id, param_id, param_value); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } @@ -4156,21 +4169,21 @@ static void ocf_mngt_cache_composite_restore_cache_lines( &begin_addr, &end_addr); ENV_BUG_ON(ret); - if (cache->device->metadata_offset >= end_addr) + if (ocf_metadata_data_offset(&cache->metadata) >= end_addr) OCF_PL_NEXT_RET(pipeline); - if (cache->device->metadata_offset > begin_addr) + if (ocf_metadata_data_offset(&cache->metadata) > begin_addr) begin_addr = 0; else - begin_addr -= cache->device->metadata_offset; + begin_addr -= ocf_metadata_data_offset(&cache->metadata); - end_addr -= cache->device->metadata_offset; + end_addr -= ocf_metadata_data_offset(&cache->metadata); begin_cline = begin_addr / ocf_cache_get_line_size(cache); end_cline = OCF_DIV_ROUND_UP(end_addr, ocf_cache_get_line_size(cache)); - if (end_cline > ocf_metadata_collision_table_entries(cache)) - end_cline = ocf_metadata_collision_table_entries(cache); + if (end_cline > ocf_metadata_line_count(&cache->metadata)) + end_cline = ocf_metadata_line_count(&cache->metadata); ocf_mngt_cache_attach_cline_range(cache, begin_cline, end_cline); @@ -4248,21 +4261,21 @@ static void ocf_mngt_detach_composite_invalidate(ocf_pipeline_t pipeline, if (result) OCF_PL_FINISH_RET(pipeline, result); - if (cache->device->metadata_offset >= end_addr) + if (ocf_metadata_data_offset(&cache->metadata) >= end_addr) OCF_PL_NEXT_RET(pipeline); - if (cache->device->metadata_offset > begin_addr) + if (ocf_metadata_data_offset(&cache->metadata) > begin_addr) begin_addr = 0; else - begin_addr -= cache->device->metadata_offset; + begin_addr -= ocf_metadata_data_offset(&cache->metadata); - end_addr -= cache->device->metadata_offset; + end_addr -= ocf_metadata_data_offset(&cache->metadata); begin_cline = begin_addr / ocf_cache_get_line_size(cache); end_cline = OCF_DIV_ROUND_UP(end_addr, ocf_cache_get_line_size(cache)); - if (end_cline > ocf_metadata_collision_table_entries(cache)) - end_cline = ocf_metadata_collision_table_entries(cache); + if (end_cline > ocf_metadata_line_count(&cache->metadata)) + end_cline = ocf_metadata_line_count(&cache->metadata); ocf_mngt_cache_detach_cline_range(cache, begin_cline, end_cline, ocf_mngt_cache_detach_composite_invalidate_cmpl, diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 089bf5373..57da32d83 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -20,6 +20,7 @@ #include "../engine/engine_common.h" #include "../ocf_seq_cutoff.h" #include "../prefetch/ocf_prefetch_priv.h" +#include "../concurrency/ocf_metadata_concurrency.h" /* Close if opened */ void cache_mngt_core_deinit(ocf_core_t core) @@ -42,11 +43,11 @@ void cache_mngt_core_remove_from_cleaning_pol(ocf_core_t core) ocf_cache_t cache = ocf_core_get_cache(core); ocf_core_id_t core_id = ocf_core_get_id(core); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); ocf_cleaning_remove_core(cache, core_id); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); } /* Deinitialize core metadata in attached metadata */ @@ -56,17 +57,19 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) ocf_core_id_t core_id = ocf_core_get_id(core); ocf_core_id_t iter_core_id; ocf_cache_line_t curr_cline, prev_cline; - uint32_t hash, num_hash = cache->device->hash_table_entries; + uint32_t hash, num_hash = ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); unsigned lock_idx; for (hash = 0; hash < num_hash;) { - prev_cline = cache->device->collision_table_entries; + prev_cline = terminator; lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue); - ocf_hb_id_prot_lock_wr(&cache->metadata.lock, lock_idx, hash); + ocf_hb_id_prot_lock_wr(&cache->metadata_lock, lock_idx, hash); curr_cline = ocf_metadata_get_hash(cache, hash); - while (curr_cline != cache->device->collision_table_entries) { + while (curr_cline != terminator) { ocf_metadata_get_core_info(cache, curr_cline, &iter_core_id, NULL); @@ -92,15 +95,15 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) ocf_cache_line_concurrency(cache), curr_cline); - if (prev_cline != cache->device->collision_table_entries) + if (prev_cline != terminator) curr_cline = ocf_metadata_get_collision_next(cache, prev_cline); else curr_cline = ocf_metadata_get_hash(cache, hash); } - ocf_hb_id_prot_unlock_wr(&cache->metadata.lock, lock_idx, hash); + ocf_hb_id_prot_unlock_wr(&cache->metadata_lock, lock_idx, hash); /* Check whether all the cachelines from the hash bucket were sparsed */ - if (curr_cline == cache->device->collision_table_entries) + if (curr_cline == terminator) hash++; else env_msleep(100); @@ -113,7 +116,7 @@ void cache_mngt_core_remove_from_meta(ocf_core_t core) ocf_cache_t cache = ocf_core_get_cache(core); ocf_core_id_t core_id = ocf_core_get_id(core); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); /* In metadata mark data this core was removed from cache */ core->conf_meta->valid = false; @@ -125,7 +128,7 @@ void cache_mngt_core_remove_from_meta(ocf_core_t core) env_bit_clear(core_id, cache->conf_meta->valid_core_bitmap); cache->conf_meta->core_count--; - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); } /* Deinit in-memory structures related to this core */ diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 58c04e58b..b0078bd60 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -21,6 +21,7 @@ #include "../utils/utils_pipeline.h" #include "../ocf_request.h" #include "../ocf_def_priv.h" +#include "../concurrency/ocf_metadata_concurrency.h" struct ocf_mngt_cache_flush_context; typedef void (*ocf_flush_complete_t)(struct ocf_mngt_cache_flush_context *, int); @@ -185,7 +186,7 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, uint32_t line, dirty_found = 0, dirty_total = 0; unsigned ret = 0; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); dirty_total = env_atomic_read(&core->runtime_meta->dirty_clines); if (!dirty_total) { @@ -201,7 +202,7 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, } for (line = 0, elem = *tbl; - line < cache->device->collision_table_entries; + line < ocf_metadata_line_count(&cache->metadata); line++) { ocf_metadata_get_core_info(cache, line, &i_core_id, &core_line); @@ -223,10 +224,10 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, if ((line + 1) % 131072 == 0) { ocf_metadata_end_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); env_cond_resched(); ocf_metadata_start_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); } } @@ -237,7 +238,7 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, *num = dirty_found; unlock: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } @@ -257,7 +258,7 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, uint32_t dirty_found = 0, dirty_total = 0, dirty_cores = 0; int ret = 0; - end = OCF_MIN(ocf_metadata_collision_table_entries(cache), end); + end = OCF_MIN(ocf_metadata_line_count(&cache->metadata), end); if (begin > end) return -OCF_ERR_INVAL; @@ -267,7 +268,7 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, return 0; } - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); core_count = cache->conf_meta->core_count; if (core_count == 0) @@ -342,10 +343,10 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, if ((line + 1) % 131072 == 0) { ocf_metadata_end_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); env_cond_resched(); ocf_metadata_start_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); } } @@ -369,7 +370,7 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, free_core_revmap: env_vfree(core_revmap); unlock: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } @@ -475,9 +476,9 @@ static int _ofc_flush_container_step(struct ocf_request *req) struct flush_container *fc = req->priv; ocf_cache_t cache = fc->cache; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); _ocf_mngt_flush_portion(fc); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return 0; } @@ -770,7 +771,7 @@ void ocf_mngt_cache_flush(ocf_cache_t cache, context->cache = cache; context->op = flush_cache; context->begin = 0; - context->end = ocf_metadata_collision_table_entries(cache); + context->end = ocf_metadata_line_count(&cache->metadata); ocf_pipeline_next(context->pipeline); } @@ -875,10 +876,10 @@ static void _ocf_mngt_cache_invalidate(ocf_pipeline_t pipeline, void *priv, ocf_cache_t cache = context->cache; int result; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_metadata_sparse_range(cache, context->purge.core_id, 0, context->purge.end_byte); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, result); } @@ -894,10 +895,10 @@ static void _ocf_mngt_cache_detach_cline_range(ocf_pipeline_t pipeline, free_detached_before = env_atomic_read(&cache->free_detached.runtime->curr_size); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_metadata_detach_cline_range(cache, context->begin, context->end); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); free_detached_after = env_atomic_read(&cache->free_detached.runtime->curr_size); @@ -921,9 +922,9 @@ int ocf_mngt_cache_attach_cline_range(ocf_cache_t cache, free_detached_before = env_atomic_read(&cache->free_detached.runtime->curr_size); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_metadata_restore_cline_range(cache, begin, end); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); free_detached_after = env_atomic_read(&cache->free_detached.runtime->curr_size); @@ -983,7 +984,7 @@ void ocf_mngt_cache_purge(ocf_cache_t cache, context->cache = cache; context->op = purge_cache; context->begin = 0; - context->end = ocf_metadata_collision_table_entries(cache); + context->end = ocf_metadata_line_count(&cache->metadata); context->purge.core_id = OCF_CORE_ID_INVALID; context->purge.end_byte = ~0ULL; @@ -1016,7 +1017,7 @@ void ocf_mngt_cache_detach_cline_range(ocf_cache_t cache, if (begin > end) OCF_CMPL_RET(cache, priv, -OCF_ERR_INVAL); - if (end > ocf_metadata_collision_table_entries(cache)) + if (end > ocf_metadata_line_count(&cache->metadata)) OCF_CMPL_RET(cache, priv, -OCF_ERR_INVAL); result = ocf_pipeline_create(&pipeline, cache, @@ -1344,11 +1345,11 @@ int ocf_mngt_cache_cleaning_set_param(ocf_cache_t cache, ocf_cleaning_t type, if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); ret = ocf_cleaning_set_param(cache, type, param_id, param_value); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } diff --git a/src/mngt/ocf_mngt_io_class.c b/src/mngt/ocf_mngt_io_class.c index b3a8cdff6..a356eb180 100644 --- a/src/mngt/ocf_mngt_io_class.c +++ b/src/mngt/ocf_mngt_io_class.c @@ -11,6 +11,7 @@ #include "../engine/cache_engine.h" #include "../utils/utils_user_part.h" #include "../ocf_lru.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "ocf_env.h" static uint64_t _ocf_mngt_count_user_parts_min_size(struct ocf_cache *cache) @@ -301,7 +302,7 @@ int ocf_mngt_cache_io_classes_configure(ocf_cache_t cache, if (!old_config) return -OCF_ERR_NO_MEM; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = env_memcpy(old_config, sizeof(cache->user_parts), cache->user_parts, sizeof(cache->user_parts)); @@ -326,7 +327,7 @@ int ocf_mngt_cache_io_classes_configure(ocf_cache_t cache, } out_cpy: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); env_free(old_config); return result; diff --git a/src/ocf_cache.c b/src/ocf_cache.c index 29e715331..189cb4085 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -148,7 +148,8 @@ int ocf_cache_get_info(ocf_cache_t cache, struct ocf_cache_info *info) info->state = cache->cache_state; info->cache_line_size = ocf_line_size(cache); info->metadata_end_offset = ocf_cache_is_device_attached(cache) ? - cache->device->metadata_offset / PAGE_SIZE : 0; + ocf_metadata_data_offset(&cache->metadata) / PAGE_SIZE : + 0; info->metadata_footprint = ocf_cache_is_device_attached(cache) ? ocf_metadata_size_of(cache) : 0; diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 11976dedc..ac0c86344 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -13,7 +13,8 @@ #include "ocf_volume_priv.h" #include "ocf_core_priv.h" #include "ocf_part.h" -#include "metadata/metadata_structs.h" +#include "metadata/metadata_priv.h" +#include "concurrency/ocf_metadata_lock.h" #include "utils/utils_list.h" #include "utils/utils_pipeline.h" #include "utils/utils_async_lock.h" @@ -31,21 +32,11 @@ struct ocf_cache_device { struct ocf_volume front_volume; struct ocf_volume volume; - /* Hash Table contains contains pointer to the entry in - * Collision Table so it actually contains collision Table - * indexes. - * Invalid entry is collision_table_entries. - */ - unsigned int hash_table_entries; - unsigned int collision_table_entries; - int metadata_error; /*!< This field indicates that an error during metadata IO * occurred */ - uint64_t metadata_offset; - struct { struct ocf_alock *cache_line; } concurrency; @@ -78,6 +69,8 @@ struct ocf_cache { struct ocf_metadata metadata; + struct ocf_metadata_lock metadata_lock; + struct { /* cache get/put counter */ struct env_refcnt cache; diff --git a/src/ocf_lru.c b/src/ocf_lru.c index af35aa2a8..27aadebc3 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -14,6 +14,7 @@ #include "utils/utils_generator.h" #include "utils/utils_parallelize.h" #include "concurrency/ocf_concurrency.h" +#include "concurrency/ocf_metadata_concurrency.h" #include "mngt/ocf_mngt_common.h" #include "engine/engine_zero.h" #include "ocf_cache_priv.h" @@ -410,7 +411,7 @@ static inline bool _lru_trylock_hash(struct ocf_lru_iter *iter, } return ocf_hb_cline_naked_trylock_wr( - &iter->cache->metadata.lock, + &iter->cache->metadata_lock, core_id, core_line); } @@ -423,7 +424,7 @@ static inline void _lru_unlock_hash(struct ocf_lru_iter *iter, } ocf_hb_cline_naked_unlock_wr( - &iter->cache->metadata.lock, + &iter->cache->metadata_lock, core_id, core_line); } @@ -486,7 +487,7 @@ static inline ocf_cache_line_t lru_req_next_cline(struct ocf_request *req, uint64_t t_core_line; ocf_part_id_t tmp_part_id; - ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_lock(&cache->metadata_lock, curr_lru); tmp_part_id = ocf_metadata_get_partition_id(cache, cline); if (tmp_part_id == PARTITION_FREELIST) @@ -532,7 +533,7 @@ static inline ocf_cache_line_t lru_req_next_cline(struct ocf_request *req, if (ret == end_marker) ocf_cache_line_unlock_wr(c, cline); lru_wr_unlock: - ocf_metadata_lru_unlock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_unlock(&cache->metadata_lock, curr_lru); return ret; } @@ -559,7 +560,7 @@ static inline ocf_cache_line_t lru_iter_eviction_next(struct ocf_lru_iter *iter, lru_iter_advance(iter); curr_lru = iter->lru_idx; - ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_lock(&cache->metadata_lock, curr_lru); list = ocf_lru_get_list(part, curr_lru, iter->clean); @@ -578,7 +579,7 @@ static inline ocf_cache_line_t lru_iter_eviction_next(struct ocf_lru_iter *iter, } } - ocf_metadata_lru_unlock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_unlock(&cache->metadata_lock, curr_lru); if (cline == end_marker && !_lru_lru_is_empty(iter)) { /* mark list as empty */ @@ -612,7 +613,7 @@ static inline ocf_cache_line_t lru_iter_free_next(struct ocf_lru_iter *iter, lru_iter_advance(iter); curr_lru = iter->lru_idx; - ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_lock(&cache->metadata_lock, curr_lru); list = ocf_lru_get_list(free, curr_lru, true); @@ -626,7 +627,7 @@ static inline ocf_cache_line_t lru_iter_free_next(struct ocf_lru_iter *iter, ocf_lru_repart_locked(cache, cline, free, dst_part); } - ocf_metadata_lru_unlock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_unlock(&cache->metadata_lock, curr_lru); if (cline == end_marker && !_lru_lru_is_empty(iter)) { /* mark list as empty */ @@ -722,7 +723,7 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, lru_idx = io_queue->lru_idx++ % OCF_NUM_LRU_LISTS; lock_idx = ocf_metadata_concurrency_next_idx(io_queue); - ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_start_shared_access(&cache->metadata_lock, lock_idx); OCF_METADATA_LRU_LOCK_ALL(); @@ -738,7 +739,7 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, OCF_METADATA_LRU_UNLOCK_ALL(); - ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_end_shared_access(&cache->metadata_lock, lock_idx); if (i == 0) { env_atomic_set(&ctx->cleaner_running, 0); @@ -1022,7 +1023,7 @@ static int ocf_lru_populate_handle(ocf_parallelize_t parallelize, struct ocf_lru_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_cache_line_t cnt, cline; - ocf_cache_line_t entries = ocf_metadata_collision_table_entries(cache); + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); struct ocf_generator_bisect_state generator; struct ocf_lru_list *list; unsigned step = 0; @@ -1166,13 +1167,13 @@ int ocf_metadata_actor(struct ocf_cache *cache, struct ocf_part *part; unsigned i, cline; struct ocf_lru_meta *node; + ocf_cache_line_t line_count = ocf_metadata_line_count(&cache->metadata); start_line = ocf_bytes_2_lines(cache, start_byte); end_line = ocf_bytes_2_lines(cache, end_byte); if (part_id == PARTITION_UNSPECIFIED) { - for (cline = 0; cline < cache->device->collision_table_entries; - ++cline) { + for (cline = 0; cline < line_count; ++cline) { if (_is_cache_line_acting(cache, cline, core_id, start_line, end_line)) { if (ocf_cache_line_is_used(c, cline)) diff --git a/src/ocf_metadata.c b/src/ocf_metadata.c index 9cc67667f..9d6b1cff7 100644 --- a/src/ocf_metadata.c +++ b/src/ocf_metadata.c @@ -11,14 +11,14 @@ static inline ocf_cache_line_t ocf_atomic_addr2line( struct ocf_cache *cache, uint64_t addr) { - addr -= cache->device->metadata_offset; + addr -= ocf_metadata_data_offset(&cache->metadata); return ocf_bytes_2_lines(cache, addr); } static inline uint8_t ocf_atomic_addr2pos(struct ocf_cache *cache, uint64_t addr) { - addr -= cache->device->metadata_offset; + addr -= ocf_metadata_data_offset(&cache->metadata); addr = BYTES_TO_BLOCKS_ROUND_DOWN(addr); addr %= ocf_line_blocks(cache); @@ -34,7 +34,7 @@ int ocf_metadata_get_atomic_entry(ocf_cache_t cache, if (addr > ocf_volume_get_length(&cache->device->volume)) return -OCF_ERR_INVAL; - if (addr < cache->device->metadata_offset) { + if (addr < ocf_metadata_data_offset(&cache->metadata)) { /* Metadata IO of OCF */ ENV_BUG_ON(env_memset(entry, sizeof(*entry), 0)); } else { @@ -71,7 +71,7 @@ int ocf_metadata_check_invalid_before(ocf_cache_t cache, uint64_t addr) line = ocf_atomic_addr2line(cache, addr); pos = ocf_atomic_addr2pos(cache, addr); - if (!pos || addr < cache->device->metadata_offset) + if (!pos || addr < ocf_metadata_data_offset(&cache->metadata)) return 0; for (i = 0; i < pos; i++) { @@ -97,7 +97,7 @@ int ocf_metadata_check_invalid_after(ocf_cache_t cache, uint64_t addr, line = ocf_atomic_addr2line(cache, addr + bytes); pos = ocf_atomic_addr2pos(cache, addr + bytes); - if (!pos || addr < cache->device->metadata_offset) + if (!pos || addr < ocf_metadata_data_offset(&cache->metadata)) return 0; for (i = pos; i < ocf_line_blocks(cache); i++) { diff --git a/src/ocf_request.h b/src/ocf_request.h index aadb6b918..867378d7d 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -11,7 +11,8 @@ #include "ocf_env.h" #include "ocf_io_priv.h" #include "ocf_def_priv.h" -#include "metadata/metadata_structs.h" +#include "metadata/metadata_priv.h" +#include "concurrency/ocf_metadata_lock.h" typedef enum { /* modes inherited from user API */ diff --git a/src/promotion/nhit/nhit.c b/src/promotion/nhit/nhit.c index 009a16117..40bf43ddd 100644 --- a/src/promotion/nhit/nhit.c +++ b/src/promotion/nhit/nhit.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2019-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -221,7 +222,7 @@ bool nhit_req_should_promote(ocf_promotion_policy_t policy, uint32_t i; uint64_t core_line; uint64_t occupied_cachelines = - ocf_metadata_collision_table_entries(policy->owner) - + ocf_metadata_line_count(&policy->owner->metadata) - ocf_lru_num_free(policy->owner); cfg = (struct nhit_promotion_policy_config*)policy->config; diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 560821025..c2f798703 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -9,6 +9,7 @@ #include "../engine/cache_engine.h" #include "../engine/engine_common.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "utils_cleaner.h" #include "utils_user_part.h" @@ -335,7 +336,7 @@ static int _ocf_cleaner_update_metadata(struct ocf_request *req) cache_line = iter->coll_idx; - ocf_hb_cline_prot_lock_wr(&cache->metadata.lock, + ocf_hb_cline_prot_lock_wr(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); @@ -351,7 +352,7 @@ static int _ocf_cleaner_update_metadata(struct ocf_request *req) cache_line); } - ocf_hb_cline_prot_unlock_wr(&cache->metadata.lock, + ocf_hb_cline_prot_unlock_wr(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); } @@ -510,13 +511,13 @@ static int _ocf_cleaner_fire_core(struct ocf_request *req) if (!iter->flush) continue; - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); _ocf_cleaner_core_submit_io(req, iter); - ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_unlock_rd(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); } @@ -573,7 +574,7 @@ static int _ocf_cleaner_fire_cache(struct ocf_request *req) addr = iter->coll_idx; addr *= ocf_line_size(cache); - addr += cache->device->metadata_offset; + addr += ocf_metadata_data_offset(&cache->metadata); offset = ocf_line_size(cache) * iter->hash; @@ -778,7 +779,8 @@ static int _ocf_cleaner_do_flush_data_getter(struct ocf_cache *cache, { struct flush_data *flush = context; - if (flush[item].cache_line < cache->device->collision_table_entries) { + if (flush[item].cache_line < + ocf_metadata_line_count(&cache->metadata)) { (*line) = flush[item].cache_line; return 0; } else { diff --git a/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c b/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c index 30badaca5..4e5cafdf6 100644 --- a/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c +++ b/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2019-2022 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -116,7 +117,7 @@ static void ocf_hb_req_prot_lock_rd_test01(void **state) const unsigned test_case_count = sizeof(test_cases) / sizeof(test_cases[0]); unsigned i; - req->cache->metadata.lock.num_hash_entries = 5; + req->cache->metadata_lock.num_hash_entries = 5; print_test_description("Verify hash locking order\n"); diff --git a/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c b/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c index 4fe9993b8..45739bf1e 100644 --- a/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c +++ b/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c @@ -38,28 +38,27 @@ static void metadata_hash_func_test01(void **state) unsigned c; ocf_core_id_t core_ids[] = {0, 1, 2, 100, OCF_CORE_NUM}; ocf_core_id_t core_id; + const uint32_t hash_entries = 10; print_test_description("Verify that hash function increments by 1 and generates" "collision after 'hash_table_entries' successive core lines"); cache = test_malloc(sizeof(*cache)); - cache->device = test_malloc(sizeof(*cache->device)); - cache->device->hash_table_entries = 10; + cache->metadata.hash_entries = hash_entries; for (c = 0; c < sizeof(core_ids) / sizeof(core_ids[0]); c++) { core_id = core_ids[c]; - for (i = 0; i < cache->device->hash_table_entries + 1; i++) { + for (i = 0; i < hash_entries + 1; i++) { hash_cur = ocf_metadata_hash_func(cache, i, core_id); hash_next = ocf_metadata_hash_func(cache, i + 1, core_id); /* make sure hash value is within expected range */ - assert(hash_cur < cache->device->hash_table_entries); - assert(hash_next < cache->device->hash_table_entries); + assert(hash_cur < hash_entries); + assert(hash_next < hash_entries); /* hash should either increment by 1 or overflow to 0 */ - assert(hash_next == (hash_cur + 1) % cache->device->hash_table_entries); + assert(hash_next == (hash_cur + 1) % hash_entries); } } - test_free(cache->device); test_free(cache); }