From 34a2413f4b2de0b51076b1e55e64a2bc68d7ddf5 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 00:30:14 +0200 Subject: [PATCH 1/8] Move metadata layout properties to metadata structure Move the fields to the metadata structure and add accessors. This allows to drop reverse dependency on ocf_cache_priv.h in metadata.h. Signed-off-by: Robert Baldyga --- src/cleaning/acp.c | 15 +++-- src/cleaning/alru.c | 63 ++++++++++++------- src/concurrency/ocf_cache_line_concurrency.c | 3 +- src/concurrency/ocf_concurrency.c | 3 +- src/engine/engine_bf.c | 6 +- src/engine/engine_common.c | 7 ++- src/engine/engine_io.c | 8 +-- src/metadata/metadata.c | 29 +++++---- src/metadata/metadata.h | 7 --- src/metadata/metadata_collision.c | 16 +++-- src/metadata/metadata_io.c | 4 +- src/metadata/metadata_misc.c | 4 +- src/metadata/metadata_misc.h | 5 +- src/metadata/metadata_raw_atomic.c | 4 +- src/metadata/metadata_structs.h | 45 +++++++++++++ src/mngt/ocf_mngt_cache.c | 28 ++++----- src/mngt/ocf_mngt_common.c | 12 ++-- src/mngt/ocf_mngt_flush.c | 10 +-- src/ocf_cache.c | 3 +- src/ocf_cache_priv.h | 10 --- src/ocf_lru.c | 6 +- src/ocf_metadata.c | 10 +-- src/promotion/nhit/nhit.c | 3 +- src/utils/utils_cleaner.c | 5 +- .../metadata_collision.c/metadata_collision.c | 13 ++-- 25 files changed, 192 insertions(+), 127 deletions(-) diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index 13c0836d7..9996d39d4 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -317,7 +317,7 @@ static int ocf_acp_populate_handle(ocf_parallelize_t parallelize, { struct ocf_acp_populate_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); ocf_cache_line_t cline, portion; uint64_t begin, end; struct acp_cleaning_policy_meta *acp_meta; @@ -479,7 +479,7 @@ void cleaning_policy_acp_populate(ocf_cache_t cache, bool reconstruct, void cleaning_policy_acp_prepopulate(ocf_cache_t cache, ocf_cleaning_op_end_t cmpl, void *priv) { - ocf_cache_line_t entries = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); ocf_cache_line_t cline; uint32_t step = 0; @@ -503,8 +503,9 @@ static int ocf_acp_update_handle(ocf_parallelize_t parallelize, { struct ocf_acp_update_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->hash_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); ocf_cache_line_t hash, cline, portion; uint64_t begin, end; unsigned lock_idx = shard_id % OCF_NUM_GLOBAL_META_LOCKS; @@ -653,7 +654,8 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, lock_idx, core_id, core_line); - return locked ? info.coll_idx : cache->device->collision_table_entries; + return locked ? info.coll_idx : + ocf_metadata_terminator_line(&cache->metadata); } static void _acp_handle_flush_error(struct ocf_cache *cache, @@ -774,7 +776,8 @@ static bool _acp_prepare_flush_data(struct acp_context *acp, ocf_cache_line_t cache_line; cache_line = _acp_trylock_dirty(cache, chunk->core_id, core_line); - if (cache_line == cache->device->collision_table_entries) + if (cache_line == + ocf_metadata_terminator_line(&cache->metadata)) continue; ACP_DEBUG_BEGIN(acp, cache_line); diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index a12f9938c..11963be96 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -71,7 +71,8 @@ struct alru_context { static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, ocf_cache_line_t head, ocf_cache_line_t tail) { - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); struct alru_cleaning_policy *part_alru; struct cleaning_policy_meta *meta; struct alru_cleaning_policy_meta *old_head; @@ -105,7 +106,8 @@ static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, static void add_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, ocf_cache_line_t cline) { - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); struct alru_cleaning_policy *part_alru; struct cleaning_policy_meta *meta; struct alru_cleaning_policy_meta *entry; @@ -133,7 +135,8 @@ static void remove_alru_list(struct ocf_cache *cache, int partition_id, unsigned int collision_index) { uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; + uint32_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id] .clean_pol->policy.alru; struct alru_cleaning_policy_meta *alru; @@ -235,7 +238,8 @@ static bool is_on_alru_list(struct ocf_cache *cache, int partition_id, unsigned int collision_index) { uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; + uint32_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id] .clean_pol->policy.alru; struct alru_cleaning_policy_meta *alru; @@ -265,8 +269,8 @@ void cleaning_policy_alru_init_cache_block(struct ocf_cache *cache, alru = &ocf_metadata_get_cleaning_policy(cache, cache_line)->meta.alru; alru->timestamp = 0; - alru->lru_prev = cache->device->collision_table_entries; - alru->lru_next = cache->device->collision_table_entries; + alru->lru_prev = ocf_metadata_terminator_line(&cache->metadata); + alru->lru_next = ocf_metadata_terminator_line(&cache->metadata); } void cleaning_policy_alru_purge_cache_block(struct ocf_cache *cache, @@ -324,7 +328,8 @@ void cleaning_policy_alru_set_hot_cache_line(struct ocf_cache *cache, struct alru_cleaning_policy *part_alru = &cache->user_parts[part_id] .clean_pol->policy.alru; uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; + uint32_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); struct alru_cleaning_policy_meta *alru; ENV_WARN_ON(!metadata_test_dirty(cache, cache_line)); @@ -419,7 +424,7 @@ static void add_alru_head_populate(struct ocf_alru_fill_context *context, struct alru_cleaning_policy_meta *entry; struct alru_cleaning_policy_meta *next; - terminator = ocf_metadata_collision_table_entries(cache); + terminator = ocf_metadata_terminator_line(&cache->metadata); curr_head = context->shard[shard_id].part[part_id].head; meta = ocf_metadata_get_cleaning_policy(cache, cline); @@ -453,8 +458,9 @@ static int ocf_alru_populate_handle(ocf_parallelize_t parallelize, { struct ocf_alru_fill_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->collision_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); unsigned part_size[OCF_USER_IO_CLASS_MAX] = {}; struct ocf_user_part *user_part; struct alru_cleaning_policy *part_alru; @@ -505,8 +511,9 @@ static int ocf_alru_prepopulate_handle(ocf_parallelize_t parallelize, { struct ocf_alru_fill_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->collision_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); unsigned part_size[OCF_USER_IO_CLASS_MAX] = {}; struct ocf_user_part *user_part; struct alru_cleaning_policy *part_alru; @@ -592,8 +599,10 @@ static void cleaning_policy_alru_fill(ocf_cache_t cache, /* ALRU initialization */ part_alru = &user_part->clean_pol->policy.alru; env_atomic_set(&part_alru->size, 0); - part_alru->lru_head = cache->device->collision_table_entries; - part_alru->lru_tail = cache->device->collision_table_entries; + part_alru->lru_head = + ocf_metadata_terminator_line(&cache->metadata); + part_alru->lru_tail = + ocf_metadata_terminator_line(&cache->metadata); cache->device->runtime_meta->cleaning_thread_access = 0; } @@ -635,8 +644,9 @@ static int ocf_alru_update_handle(ocf_parallelize_t parallelize, { struct ocf_alru_update_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t entries = cache->device->hash_table_entries; - ocf_cache_line_t terminator = cache->device->collision_table_entries; + ocf_cache_line_t entries = ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); ocf_cache_line_t hash, cline, portion; uint32_t begin, end; unsigned lock_idx = shard_id % OCF_NUM_GLOBAL_META_LOCKS; @@ -947,7 +957,7 @@ static bool more_blocks_to_flush(struct ocf_cache *cache, { struct alru_cleaning_policy_meta *alru; - if (cache_line >= cache->device->collision_table_entries) + if (cache_line >= ocf_metadata_line_count(&cache->metadata)) return false; alru = &ocf_metadata_get_cleaning_policy(cache, @@ -1000,13 +1010,18 @@ static int get_data_to_flush(struct alru_context *ctx) last_access = fctx->dirty_ratio_exceeded ? (uint32_t)(~0UL) : compute_timestamp(config); #if OCF_CLEANING_DEBUG == 1 - if (cache_line != cache->device->collision_table_entries) { - alru = &ocf_metadata_get_cleaning_policy( - cache, cache_line)->meta.alru; - OCF_DEBUG_PARAM(cache, "Last access=%u, " - "timestamp=%u rel=%d", - last_access, alru->timestamp, - alru->timestamp < last_access); + { + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); + + if (cache_line != terminator) { + alru = &ocf_metadata_get_cleaning_policy( + cache, cache_line)->meta.alru; + OCF_DEBUG_PARAM(cache, "Last access=%u, " + "timestamp=%u rel=%d", + last_access, alru->timestamp, + alru->timestamp < last_access); + } } #endif diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index c802a0a43..3729f13c6 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2022-2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -273,5 +274,5 @@ void ocf_cache_line_concurrency_deinit(struct ocf_alock **self) size_t ocf_cache_line_concurrency_size_of(ocf_cache_t cache) { - return ocf_alock_size(cache->device->collision_table_entries); + return ocf_alock_size(ocf_metadata_line_count(&cache->metadata)); } diff --git a/src/concurrency/ocf_concurrency.c b/src/concurrency/ocf_concurrency.c index 594983afa..b38172455 100644 --- a/src/concurrency/ocf_concurrency.c +++ b/src/concurrency/ocf_concurrency.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -12,7 +13,7 @@ int ocf_concurrency_init(struct ocf_cache *cache) result = ocf_cache_line_concurrency_init( &cache->device->concurrency.cache_line, - ocf_metadata_collision_table_entries(cache), + ocf_metadata_line_count(&cache->metadata), cache); if (result) diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index 34d54d4f6..bed4e600c 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -102,7 +102,7 @@ static int _ocf_backfill_do(struct ocf_request *req) ocf_req_forward_cache_init(req, _ocf_backfill_complete); if (ocf_engine_is_sequential(req)) { - addr = cache->device->metadata_offset; + addr = ocf_metadata_data_offset(&cache->metadata); addr += req->map[0].coll_idx * line_size; addr += req->addr % line_size; @@ -120,7 +120,7 @@ static int _ocf_backfill_do(struct ocf_request *req) } else { addr = req->map[i].coll_idx; addr *= line_size; - addr += cache->device->metadata_offset; + addr += ocf_metadata_data_offset(&cache->metadata); } bytes = line_size; @@ -142,7 +142,7 @@ static int _ocf_backfill_do(struct ocf_request *req) for (; i < (req->core_line_count - 1); i++) { addr_next = req->map[i + 1].coll_idx; addr_next *= line_size; - addr_next += cache->device->metadata_offset; + addr_next += ocf_metadata_data_offset(&cache->metadata); if (addr_next != (addr + bytes)) break; diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index 6868868c1..76dac7973 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -50,13 +50,13 @@ void ocf_engine_lookup_map_entry(struct ocf_cache *cache, */ entry->hash = hash; entry->status = LOOKUP_MISS; - entry->coll_idx = cache->device->collision_table_entries; + entry->coll_idx = ocf_metadata_terminator_line(&cache->metadata); entry->core_line = core_line; entry->core_id = core_id; line = ocf_metadata_get_hash(cache, hash); - while (line != cache->device->collision_table_entries) { + while (line != ocf_metadata_terminator_line(&cache->metadata)) { ocf_core_id_t curr_core_id; uint64_t curr_core_line; @@ -82,7 +82,8 @@ static inline int _ocf_engine_check_map_entry(struct ocf_cache *cache, if (entry->status == LOOKUP_MISS) return 0; - ENV_BUG_ON(entry->coll_idx >= cache->device->collision_table_entries); + ENV_BUG_ON(entry->coll_idx >= + ocf_metadata_line_count(&cache->metadata)); ocf_metadata_get_core_info(cache, entry->coll_idx, &_core_id, &_core_line); diff --git a/src/engine/engine_io.c b/src/engine/engine_io.c index faaa6af13..78e79af11 100644 --- a/src/engine/engine_io.c +++ b/src/engine/engine_io.c @@ -23,7 +23,7 @@ void ocf_engine_forward_cache_io(struct ocf_request *req, int dir, ocf_req_forward_cache_init(req, callback); - addr = cache->device->metadata_offset; + addr = ocf_metadata_data_offset(&cache->metadata); addr += req->map[first_cl].coll_idx * ocf_line_size(cache); addr += (offset + seek) % ocf_line_size(cache); @@ -44,7 +44,7 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, ocf_req_forward_cache_init(req, callback); if (ocf_engine_is_sequential(req)) { - addr = cache->device->metadata_offset; + addr = ocf_metadata_data_offset(&cache->metadata); addr += req->map[0].coll_idx * ocf_line_size(cache); addr += req->addr % ocf_line_size(cache); @@ -63,7 +63,7 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, } else { addr = req->map[i].coll_idx; addr *= ocf_line_size(cache); - addr += cache->device->metadata_offset; + addr += ocf_metadata_data_offset(&cache->metadata); } bytes = ocf_line_size(cache); @@ -78,7 +78,7 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, for (; i < (req->core_line_count - 1); i++) { addr_next = req->map[i + 1].coll_idx; addr_next *= ocf_line_size(cache); - addr_next += cache->device->metadata_offset; + addr_next += ocf_metadata_data_offset(&cache->metadata); if (addr_next != (addr + bytes)) break; diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 64d86252d..cbbcb2e57 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -167,6 +167,10 @@ void ocf_metadata_deinit_variable_size(struct ocf_cache *cache) ocf_metadata_segment_destroy(cache, ctrl->segment[i]); } ctrl->metadata_layout.pages_variable = 0; + + cache->metadata.line_count = 0; + cache->metadata.hash_entries = 0; + cache->metadata.data_offset = 0; } static inline void ocf_metadata_config_init(ocf_cache_t cache, size_t size) @@ -469,13 +473,12 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, cache->device->runtime_meta = METADATA_MEM_POOL(ctrl, metadata_segment_sb_runtime); - cache->device->collision_table_entries = - ctrl->metadata_layout.cachelines; + cache->metadata.line_count = ctrl->metadata_layout.cachelines; - cache->device->hash_table_entries = + cache->metadata.hash_entries = ctrl->raw_desc[metadata_segment_hash].layout->entries; - cache->device->metadata_offset = + cache->metadata.data_offset = ocf_metadata_layout_pages(&ctrl->metadata_layout) * PAGE_SIZE; @@ -489,7 +492,7 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, line_size / KiB); ocf_cache_log(cache, log_info, "Metadata size on device: %llu kiB\n", - cache->device->metadata_offset / KiB); + ocf_metadata_data_offset(&cache->metadata) / KiB); result = ocf_metadata_concurrency_attached_init(&cache->metadata.lock, cache, ctrl->raw_desc[metadata_segment_hash]. @@ -509,7 +512,8 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, static inline void _ocf_init_collision_entry(struct ocf_cache *cache, ocf_cache_line_t idx) { - ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; + ocf_cache_line_t invalid_idx = + ocf_metadata_terminator_line(&cache->metadata); ocf_metadata_set_collision_info(cache, idx, invalid_idx, invalid_idx); ocf_metadata_set_core_info(cache, idx, @@ -526,7 +530,8 @@ static int ocf_metadata_init_collision_handle(ocf_parallelize_t parallelize, { struct ocf_init_metadata_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t collision_table_entries = cache->device->collision_table_entries; + ocf_cache_line_t collision_table_entries = + ocf_metadata_line_count(&cache->metadata); uint32_t entry, portion, begin, end, step=0; portion = OCF_DIV_ROUND_UP((uint64_t)collision_table_entries, shards_cnt); @@ -590,8 +595,10 @@ static int ocf_metadata_init_hash_table_handle(ocf_parallelize_t parallelize, { struct ocf_init_metadata_context *context = priv; ocf_cache_t cache = context->cache; - uint32_t hash_table_entries = cache->device->hash_table_entries; - ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; + uint32_t hash_table_entries = + ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t invalid_idx = + ocf_metadata_terminator_line(&cache->metadata); uint32_t entry, portion, begin, end, step=0; portion = OCF_DIV_ROUND_UP((uint64_t)hash_table_entries, shards_cnt); @@ -1225,7 +1232,7 @@ void ocf_metadata_set_hash(struct ocf_cache *cache, ocf_cache_line_t index, { struct ocf_hash_entry *entry = ocf_metadata_get_hash_ptr(cache, index); - ENV_BUG_ON(line > cache->device->collision_table_entries); + ENV_BUG_ON(line > ocf_metadata_line_count(&cache->metadata)); entry->line = line; } @@ -1444,7 +1451,7 @@ bool ocf_metadata_is_hit_no_lock(ocf_cache_t cache, ocf_core_id_t core_id, core_line, core_id); ocf_cache_line_t line = ocf_metadata_get_hash(cache, hash); - while (line != cache->device->collision_table_entries) { + while (line != ocf_metadata_terminator_line(&cache->metadata)) { ocf_core_id_t curr_core_id; uint64_t curr_core_line; diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index 3c95933b8..6a376a148 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -9,7 +9,6 @@ #define __METADATA_H__ #include "metadata_common.h" -#include "../ocf_cache_priv.h" #include "../ocf_ctx_priv.h" #include "metadata_cleaning_policy.h" #include "metadata_lru.h" @@ -214,12 +213,6 @@ typedef void (*ocf_metadata_load_properties_end_t)(void *priv, int error, void ocf_metadata_load_properties(ocf_volume_t volume, ocf_metadata_load_properties_end_t cmpl, void *priv); -static inline ocf_cache_line_t ocf_metadata_collision_table_entries( - struct ocf_cache *cache) -{ - return cache->device->collision_table_entries; -} - void ocf_metadata_zero_superblock(ocf_cache_t cache, ocf_metadata_end_t cmpl, void *context); diff --git a/src/metadata/metadata_collision.c b/src/metadata/metadata_collision.c index 0e4cd1875..a9c2baf34 100644 --- a/src/metadata/metadata_collision.c +++ b/src/metadata/metadata_collision.c @@ -172,9 +172,9 @@ void ocf_metadata_get_collision_info(struct ocf_cache *cache, ocf_metadata_error(cache); if (next) - *next = cache->device->collision_table_entries; + *next = ocf_metadata_terminator_line(&cache->metadata); if (prev) - *prev = cache->device->collision_table_entries; + *prev = ocf_metadata_terminator_line(&cache->metadata); } } @@ -187,8 +187,10 @@ void ocf_metadata_add_to_collision(struct ocf_cache *cache, ocf_cache_line_t hash, ocf_cache_line_t cache_line) { ocf_cache_line_t prev_cache_line = ocf_metadata_get_hash(cache, hash); - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - ocf_cache_line_t hash_entries = cache->device->hash_table_entries; + ocf_cache_line_t line_entries = + ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t hash_entries = + ocf_metadata_hash_entries(&cache->metadata); ENV_BUG_ON(!(hash < hash_entries)); ENV_BUG_ON(!(cache_line < line_entries)); @@ -226,8 +228,10 @@ void ocf_metadata_remove_from_collision(struct ocf_cache *cache, uint64_t core_line; ocf_cache_line_t hash_father; ocf_cache_line_t prev_line, next_line; - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - ocf_cache_line_t hash_entries = cache->device->hash_table_entries; + ocf_cache_line_t line_entries = + ocf_metadata_line_count(&cache->metadata); + ocf_cache_line_t hash_entries = + ocf_metadata_hash_entries(&cache->metadata); ENV_BUG_ON(!(line < line_entries)); diff --git a/src/metadata/metadata_io.c b/src/metadata/metadata_io.c index 7145d5c6b..fad040ce4 100644 --- a/src/metadata/metadata_io.c +++ b/src/metadata/metadata_io.c @@ -115,7 +115,7 @@ static int metadata_io_read_i_atomic_step(struct ocf_request *req) ocf_req_forward_cache_init(req, metadata_io_read_i_atomic_step_end); ocf_req_forward_cache_metadata(req, OCF_READ, - cache->device->metadata_offset + + ocf_metadata_data_offset(&cache->metadata) + PAGES_TO_BYTES(context->curr_offset), PAGES_TO_BYTES(context->curr_count), 0); @@ -130,7 +130,7 @@ int metadata_io_read_i_atomic(ocf_cache_t cache, ocf_queue_t queue, void *priv, ocf_metadata_io_end_t compl_hndl) { struct metadata_io_read_i_atomic_context *context; - uint64_t io_sectors_count = cache->device->collision_table_entries * + uint64_t io_sectors_count = ocf_metadata_line_count(&cache->metadata) * ocf_line_blocks(cache); struct ocf_request *req; diff --git a/src/metadata/metadata_misc.c b/src/metadata/metadata_misc.c index e5906acc7..047385498 100644 --- a/src/metadata/metadata_misc.c +++ b/src/metadata/metadata_misc.c @@ -57,7 +57,7 @@ int ocf_metadata_detach_cline_range(ocf_cache_t cache, ocf_cache_line_t begin, if (begin > end) return -OCF_ERR_INVAL; - if (end > cache->device->collision_table_entries) + if (end > ocf_metadata_line_count(&cache->metadata)) return -OCF_ERR_INVAL; for (cline = begin; cline < end; ++cline) { @@ -90,7 +90,7 @@ int ocf_metadata_restore_cline_range(ocf_cache_t cache, ocf_cache_line_t begin, if (begin > end) return -OCF_ERR_INVAL; - if (end > cache->device->collision_table_entries) + if (end > ocf_metadata_line_count(&cache->metadata)) return -OCF_ERR_INVAL; for (cline = begin; cline < end; ++cline) { diff --git a/src/metadata/metadata_misc.h b/src/metadata/metadata_misc.h index 6e1636c1c..acf493e24 100644 --- a/src/metadata/metadata_misc.h +++ b/src/metadata/metadata_misc.h @@ -14,7 +14,8 @@ static inline ocf_cache_line_t ocf_metadata_hash_func(ocf_cache_t cache, uint64_t core_line_num, ocf_core_id_t core_id) { - return ocf_metadata_collision_hash(cache->device->hash_table_entries, + return ocf_metadata_collision_hash( + ocf_metadata_hash_entries(&cache->metadata), core_line_num, core_id); } @@ -23,7 +24,7 @@ static inline ocf_cache_line_t ocf_metadata_hash_next(ocf_cache_t cache, ocf_cache_line_t hash) { return ocf_metadata_collision_hash_next( - cache->device->hash_table_entries, hash); + ocf_metadata_hash_entries(&cache->metadata), hash); } void ocf_metadata_remove_cache_line(struct ocf_cache *cache, diff --git a/src/metadata/metadata_raw_atomic.c b/src/metadata/metadata_raw_atomic.c index eb7eb8621..e41b8554e 100644 --- a/src/metadata/metadata_raw_atomic.c +++ b/src/metadata/metadata_raw_atomic.c @@ -89,7 +89,7 @@ static void _raw_atomic_flush_do_asynch_sec(struct ocf_cache *cache, start_addr = map->coll_idx; start_addr *= ocf_line_size(cache); - start_addr += cache->device->metadata_offset; + start_addr += ocf_metadata_data_offset(&cache->metadata); start_addr += BLOCKS_TO_BYTES(map->start_flush); len = BLOCKS_TO_BYTES(map->stop_flush - map->start_flush); @@ -179,7 +179,7 @@ int raw_atomic_flush_do_asynch(struct ocf_cache *cache, struct ocf_request *req, while (i < clines_to_flush) { start_addr = clines_tab[i]; start_addr *= ocf_line_size(cache); - start_addr += cache->device->metadata_offset; + start_addr += ocf_metadata_data_offset(&cache->metadata); len = ocf_line_size(cache); while (true) { diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index 3d2693d63..0d00b0476 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -52,7 +52,52 @@ struct ocf_metadata { bool is_volatile; /*!< true if metadata used in volatile mode (RAM only) */ + ocf_cache_line_t line_count; + /*!< Number of collision table entries */ + + uint32_t hash_entries; + /*!< Number of hash table entries */ + + uint64_t data_offset; + /*!< Start of cache data area on cache device (in bytes) */ + struct ocf_metadata_lock lock; }; +/** + * @brief Get number of cache lines (collision table entries) + */ +static inline ocf_cache_line_t ocf_metadata_line_count( + const struct ocf_metadata *metadata) +{ + return metadata->line_count; +} + +/** + * @brief Get cache line index used as list terminator / unmapped marker + */ +static inline ocf_cache_line_t ocf_metadata_terminator_line( + const struct ocf_metadata *metadata) +{ + return metadata->line_count; +} + +/** + * @brief Get number of hash table entries + */ +static inline uint32_t ocf_metadata_hash_entries( + const struct ocf_metadata *metadata) +{ + return metadata->hash_entries; +} + +/** + * @brief Get offset of cache data area on cache device (in bytes) + */ +static inline uint64_t ocf_metadata_data_offset( + const struct ocf_metadata *metadata) +{ + return metadata->data_offset; +} + #endif /* __METADATA_STRUCTS_H__ */ diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index ffdc396c0..c765aa637 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -621,7 +621,7 @@ static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize, ocf_core_id_t core_id; uint64_t core_line; unsigned char step = 0; - const uint64_t entries = ocf_metadata_collision_table_entries(cache); + const uint64_t entries = ocf_metadata_line_count(&cache->metadata); begin = shard_id * OCF_LRU_CHUNK_SIZE; @@ -1961,7 +1961,7 @@ static void _ocf_mngt_attach_discard(ocf_pipeline_t pipeline, { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - uint64_t addr = cache->device->metadata_offset; + uint64_t addr = ocf_metadata_data_offset(&cache->metadata); uint64_t length = ocf_volume_get_length(&cache->device->volume) - addr; bool discard = cache->device->volume.features.discard_zeroes; @@ -4156,21 +4156,21 @@ static void ocf_mngt_cache_composite_restore_cache_lines( &begin_addr, &end_addr); ENV_BUG_ON(ret); - if (cache->device->metadata_offset >= end_addr) + if (ocf_metadata_data_offset(&cache->metadata) >= end_addr) OCF_PL_NEXT_RET(pipeline); - if (cache->device->metadata_offset > begin_addr) + if (ocf_metadata_data_offset(&cache->metadata) > begin_addr) begin_addr = 0; else - begin_addr -= cache->device->metadata_offset; + begin_addr -= ocf_metadata_data_offset(&cache->metadata); - end_addr -= cache->device->metadata_offset; + end_addr -= ocf_metadata_data_offset(&cache->metadata); begin_cline = begin_addr / ocf_cache_get_line_size(cache); end_cline = OCF_DIV_ROUND_UP(end_addr, ocf_cache_get_line_size(cache)); - if (end_cline > ocf_metadata_collision_table_entries(cache)) - end_cline = ocf_metadata_collision_table_entries(cache); + if (end_cline > ocf_metadata_line_count(&cache->metadata)) + end_cline = ocf_metadata_line_count(&cache->metadata); ocf_mngt_cache_attach_cline_range(cache, begin_cline, end_cline); @@ -4248,21 +4248,21 @@ static void ocf_mngt_detach_composite_invalidate(ocf_pipeline_t pipeline, if (result) OCF_PL_FINISH_RET(pipeline, result); - if (cache->device->metadata_offset >= end_addr) + if (ocf_metadata_data_offset(&cache->metadata) >= end_addr) OCF_PL_NEXT_RET(pipeline); - if (cache->device->metadata_offset > begin_addr) + if (ocf_metadata_data_offset(&cache->metadata) > begin_addr) begin_addr = 0; else - begin_addr -= cache->device->metadata_offset; + begin_addr -= ocf_metadata_data_offset(&cache->metadata); - end_addr -= cache->device->metadata_offset; + end_addr -= ocf_metadata_data_offset(&cache->metadata); begin_cline = begin_addr / ocf_cache_get_line_size(cache); end_cline = OCF_DIV_ROUND_UP(end_addr, ocf_cache_get_line_size(cache)); - if (end_cline > ocf_metadata_collision_table_entries(cache)) - end_cline = ocf_metadata_collision_table_entries(cache); + if (end_cline > ocf_metadata_line_count(&cache->metadata)) + end_cline = ocf_metadata_line_count(&cache->metadata); ocf_mngt_cache_detach_cline_range(cache, begin_cline, end_cline, ocf_mngt_cache_detach_composite_invalidate_cmpl, diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 089bf5373..316edb628 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -56,17 +56,19 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) ocf_core_id_t core_id = ocf_core_get_id(core); ocf_core_id_t iter_core_id; ocf_cache_line_t curr_cline, prev_cline; - uint32_t hash, num_hash = cache->device->hash_table_entries; + uint32_t hash, num_hash = ocf_metadata_hash_entries(&cache->metadata); + ocf_cache_line_t terminator = + ocf_metadata_terminator_line(&cache->metadata); unsigned lock_idx; for (hash = 0; hash < num_hash;) { - prev_cline = cache->device->collision_table_entries; + prev_cline = terminator; lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue); ocf_hb_id_prot_lock_wr(&cache->metadata.lock, lock_idx, hash); curr_cline = ocf_metadata_get_hash(cache, hash); - while (curr_cline != cache->device->collision_table_entries) { + while (curr_cline != terminator) { ocf_metadata_get_core_info(cache, curr_cline, &iter_core_id, NULL); @@ -92,7 +94,7 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) ocf_cache_line_concurrency(cache), curr_cline); - if (prev_cline != cache->device->collision_table_entries) + if (prev_cline != terminator) curr_cline = ocf_metadata_get_collision_next(cache, prev_cline); else curr_cline = ocf_metadata_get_hash(cache, hash); @@ -100,7 +102,7 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) ocf_hb_id_prot_unlock_wr(&cache->metadata.lock, lock_idx, hash); /* Check whether all the cachelines from the hash bucket were sparsed */ - if (curr_cline == cache->device->collision_table_entries) + if (curr_cline == terminator) hash++; else env_msleep(100); diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 58c04e58b..6e93451fe 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -201,7 +201,7 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, } for (line = 0, elem = *tbl; - line < cache->device->collision_table_entries; + line < ocf_metadata_line_count(&cache->metadata); line++) { ocf_metadata_get_core_info(cache, line, &i_core_id, &core_line); @@ -257,7 +257,7 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, uint32_t dirty_found = 0, dirty_total = 0, dirty_cores = 0; int ret = 0; - end = OCF_MIN(ocf_metadata_collision_table_entries(cache), end); + end = OCF_MIN(ocf_metadata_line_count(&cache->metadata), end); if (begin > end) return -OCF_ERR_INVAL; @@ -770,7 +770,7 @@ void ocf_mngt_cache_flush(ocf_cache_t cache, context->cache = cache; context->op = flush_cache; context->begin = 0; - context->end = ocf_metadata_collision_table_entries(cache); + context->end = ocf_metadata_line_count(&cache->metadata); ocf_pipeline_next(context->pipeline); } @@ -983,7 +983,7 @@ void ocf_mngt_cache_purge(ocf_cache_t cache, context->cache = cache; context->op = purge_cache; context->begin = 0; - context->end = ocf_metadata_collision_table_entries(cache); + context->end = ocf_metadata_line_count(&cache->metadata); context->purge.core_id = OCF_CORE_ID_INVALID; context->purge.end_byte = ~0ULL; @@ -1016,7 +1016,7 @@ void ocf_mngt_cache_detach_cline_range(ocf_cache_t cache, if (begin > end) OCF_CMPL_RET(cache, priv, -OCF_ERR_INVAL); - if (end > ocf_metadata_collision_table_entries(cache)) + if (end > ocf_metadata_line_count(&cache->metadata)) OCF_CMPL_RET(cache, priv, -OCF_ERR_INVAL); result = ocf_pipeline_create(&pipeline, cache, diff --git a/src/ocf_cache.c b/src/ocf_cache.c index 29e715331..189cb4085 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -148,7 +148,8 @@ int ocf_cache_get_info(ocf_cache_t cache, struct ocf_cache_info *info) info->state = cache->cache_state; info->cache_line_size = ocf_line_size(cache); info->metadata_end_offset = ocf_cache_is_device_attached(cache) ? - cache->device->metadata_offset / PAGE_SIZE : 0; + ocf_metadata_data_offset(&cache->metadata) / PAGE_SIZE : + 0; info->metadata_footprint = ocf_cache_is_device_attached(cache) ? ocf_metadata_size_of(cache) : 0; diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 11976dedc..9ed18a7bf 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -31,21 +31,11 @@ struct ocf_cache_device { struct ocf_volume front_volume; struct ocf_volume volume; - /* Hash Table contains contains pointer to the entry in - * Collision Table so it actually contains collision Table - * indexes. - * Invalid entry is collision_table_entries. - */ - unsigned int hash_table_entries; - unsigned int collision_table_entries; - int metadata_error; /*!< This field indicates that an error during metadata IO * occurred */ - uint64_t metadata_offset; - struct { struct ocf_alock *cache_line; } concurrency; diff --git a/src/ocf_lru.c b/src/ocf_lru.c index af35aa2a8..349c5b801 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -1022,7 +1022,7 @@ static int ocf_lru_populate_handle(ocf_parallelize_t parallelize, struct ocf_lru_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_cache_line_t cnt, cline; - ocf_cache_line_t entries = ocf_metadata_collision_table_entries(cache); + ocf_cache_line_t entries = ocf_metadata_line_count(&cache->metadata); struct ocf_generator_bisect_state generator; struct ocf_lru_list *list; unsigned step = 0; @@ -1166,13 +1166,13 @@ int ocf_metadata_actor(struct ocf_cache *cache, struct ocf_part *part; unsigned i, cline; struct ocf_lru_meta *node; + ocf_cache_line_t line_count = ocf_metadata_line_count(&cache->metadata); start_line = ocf_bytes_2_lines(cache, start_byte); end_line = ocf_bytes_2_lines(cache, end_byte); if (part_id == PARTITION_UNSPECIFIED) { - for (cline = 0; cline < cache->device->collision_table_entries; - ++cline) { + for (cline = 0; cline < line_count; ++cline) { if (_is_cache_line_acting(cache, cline, core_id, start_line, end_line)) { if (ocf_cache_line_is_used(c, cline)) diff --git a/src/ocf_metadata.c b/src/ocf_metadata.c index 9cc67667f..9d6b1cff7 100644 --- a/src/ocf_metadata.c +++ b/src/ocf_metadata.c @@ -11,14 +11,14 @@ static inline ocf_cache_line_t ocf_atomic_addr2line( struct ocf_cache *cache, uint64_t addr) { - addr -= cache->device->metadata_offset; + addr -= ocf_metadata_data_offset(&cache->metadata); return ocf_bytes_2_lines(cache, addr); } static inline uint8_t ocf_atomic_addr2pos(struct ocf_cache *cache, uint64_t addr) { - addr -= cache->device->metadata_offset; + addr -= ocf_metadata_data_offset(&cache->metadata); addr = BYTES_TO_BLOCKS_ROUND_DOWN(addr); addr %= ocf_line_blocks(cache); @@ -34,7 +34,7 @@ int ocf_metadata_get_atomic_entry(ocf_cache_t cache, if (addr > ocf_volume_get_length(&cache->device->volume)) return -OCF_ERR_INVAL; - if (addr < cache->device->metadata_offset) { + if (addr < ocf_metadata_data_offset(&cache->metadata)) { /* Metadata IO of OCF */ ENV_BUG_ON(env_memset(entry, sizeof(*entry), 0)); } else { @@ -71,7 +71,7 @@ int ocf_metadata_check_invalid_before(ocf_cache_t cache, uint64_t addr) line = ocf_atomic_addr2line(cache, addr); pos = ocf_atomic_addr2pos(cache, addr); - if (!pos || addr < cache->device->metadata_offset) + if (!pos || addr < ocf_metadata_data_offset(&cache->metadata)) return 0; for (i = 0; i < pos; i++) { @@ -97,7 +97,7 @@ int ocf_metadata_check_invalid_after(ocf_cache_t cache, uint64_t addr, line = ocf_atomic_addr2line(cache, addr + bytes); pos = ocf_atomic_addr2pos(cache, addr + bytes); - if (!pos || addr < cache->device->metadata_offset) + if (!pos || addr < ocf_metadata_data_offset(&cache->metadata)) return 0; for (i = pos; i < ocf_line_blocks(cache); i++) { diff --git a/src/promotion/nhit/nhit.c b/src/promotion/nhit/nhit.c index 009a16117..40bf43ddd 100644 --- a/src/promotion/nhit/nhit.c +++ b/src/promotion/nhit/nhit.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2019-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -221,7 +222,7 @@ bool nhit_req_should_promote(ocf_promotion_policy_t policy, uint32_t i; uint64_t core_line; uint64_t occupied_cachelines = - ocf_metadata_collision_table_entries(policy->owner) - + ocf_metadata_line_count(&policy->owner->metadata) - ocf_lru_num_free(policy->owner); cfg = (struct nhit_promotion_policy_config*)policy->config; diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 560821025..82882feea 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -573,7 +573,7 @@ static int _ocf_cleaner_fire_cache(struct ocf_request *req) addr = iter->coll_idx; addr *= ocf_line_size(cache); - addr += cache->device->metadata_offset; + addr += ocf_metadata_data_offset(&cache->metadata); offset = ocf_line_size(cache) * iter->hash; @@ -778,7 +778,8 @@ static int _ocf_cleaner_do_flush_data_getter(struct ocf_cache *cache, { struct flush_data *flush = context; - if (flush[item].cache_line < cache->device->collision_table_entries) { + if (flush[item].cache_line < + ocf_metadata_line_count(&cache->metadata)) { (*line) = flush[item].cache_line; return 0; } else { diff --git a/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c b/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c index 4fe9993b8..45739bf1e 100644 --- a/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c +++ b/tests/unit/tests/metadata/metadata_collision.c/metadata_collision.c @@ -38,28 +38,27 @@ static void metadata_hash_func_test01(void **state) unsigned c; ocf_core_id_t core_ids[] = {0, 1, 2, 100, OCF_CORE_NUM}; ocf_core_id_t core_id; + const uint32_t hash_entries = 10; print_test_description("Verify that hash function increments by 1 and generates" "collision after 'hash_table_entries' successive core lines"); cache = test_malloc(sizeof(*cache)); - cache->device = test_malloc(sizeof(*cache->device)); - cache->device->hash_table_entries = 10; + cache->metadata.hash_entries = hash_entries; for (c = 0; c < sizeof(core_ids) / sizeof(core_ids[0]); c++) { core_id = core_ids[c]; - for (i = 0; i < cache->device->hash_table_entries + 1; i++) { + for (i = 0; i < hash_entries + 1; i++) { hash_cur = ocf_metadata_hash_func(cache, i, core_id); hash_next = ocf_metadata_hash_func(cache, i + 1, core_id); /* make sure hash value is within expected range */ - assert(hash_cur < cache->device->hash_table_entries); - assert(hash_next < cache->device->hash_table_entries); + assert(hash_cur < hash_entries); + assert(hash_next < hash_entries); /* hash should either increment by 1 or overflow to 0 */ - assert(hash_next == (hash_cur + 1) % cache->device->hash_table_entries); + assert(hash_next == (hash_cur + 1) % hash_entries); } } - test_free(cache->device); test_free(cache); } From a6deead7f813fcc183fe0573baa742986a4c6600 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 01:13:20 +0200 Subject: [PATCH 2/8] Remove global metadata lock from metadata page fill Collision segment pages flushed at runtime are already protected by collision page locks, which keep each cache line consistent. All other segments are flushed only from management operations, and nothing changes their content concurrently. The shared global lock adds no protection, and it delays I/O-path metadata flushes behind exclusive management sections. Signed-off-by: Robert Baldyga --- src/metadata/metadata_io.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/metadata/metadata_io.c b/src/metadata/metadata_io.c index fad040ce4..6fd629140 100644 --- a/src/metadata/metadata_io.c +++ b/src/metadata/metadata_io.c @@ -207,14 +207,14 @@ static int metadata_io_do(struct ocf_request *req) ctx_data_seek(cache->owner, req->data, ctx_data_seek_begin, 0); - /* Fill with the latest metadata. */ - if (m_req->req.rw == OCF_WRITE) { - ocf_metadata_start_shared_access(&cache->metadata.lock, - m_req->page % OCF_NUM_GLOBAL_META_LOCKS); + /* + * Fill with the latest metadata. Collision segment pages, flushed + * at runtime, are synchronized by the segment page lock. Other + * segments are flushed only from management operations, during which + * their content is not modified concurrently. + */ + if (m_req->req.rw == OCF_WRITE) metadata_io_req_fill(m_req); - ocf_metadata_end_shared_access(&cache->metadata.lock, - m_req->page % OCF_NUM_GLOBAL_META_LOCKS); - } ctx_data_seek(cache->owner, req->data, ctx_data_seek_begin, 0); From 237c665bece698bbb6421e440db41b7413d44edc Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 12:52:38 +0200 Subject: [PATCH 3/8] metadata: Move collision page locks to metadata_raw Page locks keep pages of metadata flushed at runtime consistent. They are a property of how a metadata segment is persisted, not of cache concurrency. Let raw own them, and drop the page lock callbacks and the collision lock interface from metadata concurrency. Signed-off-by: Robert Baldyga --- src/cleaning/alru.c | 1 + src/concurrency/ocf_metadata_concurrency.c | 77 +------------------ src/concurrency/ocf_metadata_concurrency.h | 17 +---- src/engine/engine_bf.c | 1 + src/engine/engine_common.c | 1 + src/engine/engine_discard.c | 1 + src/engine/engine_fast.c | 1 + src/engine/engine_inv.c | 1 + src/engine/engine_prefetch.c | 1 + src/engine/engine_pt.c | 1 + src/engine/engine_rd.c | 1 + src/engine/engine_wa.c | 1 + src/engine/engine_wb.c | 1 + src/engine/engine_wi.c | 1 + src/engine/engine_wo.c | 1 + src/engine/engine_wt.c | 1 + src/engine/engine_zero.c | 1 + src/metadata/metadata.c | 47 ++---------- src/metadata/metadata_collision.c | 4 +- src/metadata/metadata_passive_update.c | 1 - src/metadata/metadata_raw.c | 88 +++++++++++++++++----- src/metadata/metadata_raw.h | 73 +++++++++++++++--- src/metadata/metadata_raw_dynamic.c | 16 +--- src/metadata/metadata_raw_dynamic.h | 4 +- src/metadata/metadata_segment.c | 11 +-- src/metadata/metadata_segment.h | 7 +- src/metadata/metadata_status.h | 2 +- src/metadata/metadata_structs.h | 2 - src/metadata/metadata_superblock.c | 2 +- src/mngt/ocf_mngt_common.c | 1 + src/mngt/ocf_mngt_flush.c | 1 + src/mngt/ocf_mngt_io_class.c | 1 + src/ocf_lru.c | 1 + src/utils/utils_cleaner.c | 1 + 34 files changed, 176 insertions(+), 195 deletions(-) diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index 11963be96..be8e40fd1 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -16,6 +16,7 @@ #include "../utils/utils_parallelize.h" #include "../utils/utils_realloc.h" #include "../concurrency/ocf_cache_line_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_def_priv.h" #include "cleaning_priv.h" diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c index f3fdc1f60..53c6fcd8b 100644 --- a/src/concurrency/ocf_metadata_concurrency.c +++ b/src/concurrency/ocf_metadata_concurrency.c @@ -128,57 +128,12 @@ static inline void hash_lock_unlock(struct ocf_cache *cache, int index, int rw) } } -int ocf_metadata_concurrency_attached_init( +void ocf_metadata_concurrency_attached_init( struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache, - uint32_t hash_table_entries, uint32_t colision_table_pages) + uint32_t hash_table_entries) { - uint32_t i; - int err = 0; - metadata_lock->cache = cache; metadata_lock->num_hash_entries = hash_table_entries; - - if (cache->metadata.is_volatile || colision_table_pages == 0) { - metadata_lock->collision_pages = NULL; - metadata_lock->num_collision_pages = 0; - return 0; - } - - metadata_lock->collision_pages = env_vzalloc(sizeof(env_rwsem) * - colision_table_pages); - if (!metadata_lock->collision_pages) - return -OCF_ERR_NO_MEM; - - for (i = 0; i < colision_table_pages; i++) { - err = env_rwsem_init(&metadata_lock->collision_pages[i]); - if (err) - break; - } - if (err) { - while (i--) - env_rwsem_destroy(&metadata_lock->collision_pages[i]); - env_vfree(metadata_lock->collision_pages); - metadata_lock->collision_pages = NULL; - return err; - } - - metadata_lock->num_collision_pages = colision_table_pages; - - return 0; -} - -void ocf_metadata_concurrency_attached_deinit( - struct ocf_metadata_lock *metadata_lock) -{ - uint32_t i; - - if (metadata_lock->collision_pages) { - for (i = 0; i < metadata_lock->num_collision_pages; i++) - env_rwsem_destroy(&metadata_lock->collision_pages[i]); - env_vfree(metadata_lock->collision_pages); - metadata_lock->collision_pages = NULL; - metadata_lock->num_collision_pages = 0; - } } void ocf_metadata_start_exclusive_access( @@ -550,31 +505,3 @@ void ocf_hb_req_prot_unlock_wr(struct ocf_request *req) ocf_metadata_end_shared_access(&req->cache->metadata.lock, req->lock_idx); } - -void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_down_read(&metadata_lock->collision_pages[page]); -} - -void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_up_read(&metadata_lock->collision_pages[page]); -} - -void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_down_write(&metadata_lock->collision_pages[page]); -} - -void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page) -{ - if (metadata_lock->collision_pages) - env_rwsem_up_write(&metadata_lock->collision_pages[page]); -} diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h index c4db69bad..3a9fb5cb1 100644 --- a/src/concurrency/ocf_metadata_concurrency.h +++ b/src/concurrency/ocf_metadata_concurrency.h @@ -23,12 +23,9 @@ int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock); void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock); -int ocf_metadata_concurrency_attached_init( +void ocf_metadata_concurrency_attached_init( struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache, - uint32_t hash_table_entries, uint32_t colision_table_pages); - -void ocf_metadata_concurrency_attached_deinit( - struct ocf_metadata_lock *metadata_lock); + uint32_t hash_table_entries); static inline void ocf_metadata_lru_lock( struct ocf_metadata_lock *metadata_lock, unsigned ev_list) @@ -154,13 +151,5 @@ void ocf_hb_req_prot_lock_wr(struct ocf_request *req); void ocf_hb_req_prot_unlock_wr(struct ocf_request *req); void ocf_hb_req_prot_lock_upgrade(struct ocf_request *req); -/* collision table page lock interface */ -void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); -void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); -void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); -void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock, - uint32_t page); + #endif /* __OCF_METADATA_CONCURRENCY_H__ */ diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index bed4e600c..6eb302901 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -15,6 +15,7 @@ #include "cache_engine.h" #include "../ocf_request.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../utils/utils_io.h" #define OCF_ENGINE_DEBUG_IO_NAME "bf" diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index 76dac7973..77803ab2d 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -20,6 +20,7 @@ #include "../ocf_space.h" #include "../promotion/promotion.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" void ocf_engine_error(struct ocf_request *req, bool stop_cache, const char *msg) diff --git a/src/engine/engine_discard.c b/src/engine/engine_discard.c index 20bdcc33d..6c5047608 100644 --- a/src/engine/engine_discard.c +++ b/src/engine/engine_discard.c @@ -14,6 +14,7 @@ #include "../ocf_request.h" #include "../utils/utils_io.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG 0 diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index b35788e5c..5a687ed0f 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -14,6 +14,7 @@ #include "../ocf_request.h" #include "../utils/utils_user_part.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../metadata/metadata.h" #define OCF_ENGINE_DEBUG 0 diff --git a/src/engine/engine_inv.c b/src/engine/engine_inv.c index 7a9ec5ccd..0d70ec77b 100644 --- a/src/engine/engine_inv.c +++ b/src/engine/engine_inv.c @@ -13,6 +13,7 @@ #include "../utils/utils_cache_line.h" #include "../metadata/metadata.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "inv" #include "engine_debug.h" diff --git a/src/engine/engine_prefetch.c b/src/engine/engine_prefetch.c index ceb4bf34c..d6e74a9d7 100644 --- a/src/engine/engine_prefetch.c +++ b/src/engine/engine_prefetch.c @@ -13,6 +13,7 @@ #include "engine_io.h" #include "../ocf_cache_priv.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index f15c05594..0e62fecc0 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -14,6 +14,7 @@ #include "../utils/utils_user_part.h" #include "../metadata/metadata.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "pt" #include "engine_debug.h" diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index ac994fa05..95729f484 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -14,6 +14,7 @@ #include "engine_io.h" #include "cache_engine.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" diff --git a/src/engine/engine_wa.c b/src/engine/engine_wa.c index c78131863..ee1ea5332 100644 --- a/src/engine/engine_wa.c +++ b/src/engine/engine_wa.c @@ -13,6 +13,7 @@ #include "cache_engine.h" #include "../ocf_request.h" #include "../metadata/metadata.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wa" #include "engine_debug.h" diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 1860de467..025e8cfed 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -18,6 +18,7 @@ #include "../utils/utils_request.h" #include "../utils/utils_user_part.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wb" #include "engine_debug.h" diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index ff836fd2a..6ff298f5c 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -11,6 +11,7 @@ #include "engine_common.h" #include "engine_io.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../metadata/metadata.h" diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index 9d771cd2c..50d4c635d 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -17,6 +17,7 @@ #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wo" #include "engine_debug.h" diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index 303dc0773..8bd51f3c6 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -16,6 +16,7 @@ #include "../utils/utils_user_part.h" #include "../metadata/metadata.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #define OCF_ENGINE_DEBUG_IO_NAME "wt" #include "engine_debug.h" diff --git a/src/engine/engine_zero.c b/src/engine/engine_zero.c index 0a74216f3..c5d241723 100644 --- a/src/engine/engine_zero.c +++ b/src/engine/engine_zero.c @@ -10,6 +10,7 @@ #include "engine_zero.h" #include "engine_common.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_io.h" diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index cbbcb2e57..ce696be06 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -17,6 +17,7 @@ #include "metadata_raw.h" #include "metadata_segment.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" #include "../utils/utils_cache_line.h" @@ -157,8 +158,6 @@ void ocf_metadata_deinit_variable_size(struct ocf_cache *cache) OCF_DEBUG_TRACE(cache); - ocf_metadata_concurrency_attached_deinit(&cache->metadata.lock); - /* * De initialize RAW types */ @@ -285,7 +284,7 @@ static int ocf_metadata_init_fixed_size(struct ocf_cache *cache, &ctrl->segment[i], cache, &ctrl->raw_desc[i], - NULL, NULL, + false, superblock); if (result) break; @@ -324,23 +323,6 @@ static int ocf_metadata_init_fixed_size(struct ocf_cache *cache, return 0; } -static void ocf_metadata_flush_lock_collision_page(struct ocf_cache *cache, - struct ocf_metadata_raw *raw, uint32_t page) - -{ - ocf_collision_start_exclusive_access(&cache->metadata.lock, - page); -} - -static void ocf_metadata_flush_unlock_collision_page( - struct ocf_cache *cache, struct ocf_metadata_raw *raw, - uint32_t page) - -{ - ocf_collision_end_exclusive_access(&cache->metadata.lock, - page); -} - /* * Initialize hash metadata interface */ @@ -351,7 +333,6 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, int result = 0; uint32_t i = 0; struct ocf_metadata_ctrl *ctrl = NULL; - ocf_flush_page_synch_t lock_page, unlock_page; uint64_t device_lines; struct ocf_metadata_segment *superblock; @@ -418,20 +399,12 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, if (raw->layout->disabled) continue; - if (i == metadata_segment_collision) { - lock_page = - ocf_metadata_flush_lock_collision_page; - unlock_page = - ocf_metadata_flush_unlock_collision_page; - } else { - lock_page = unlock_page = NULL; - } - + /* Collision segment is flushed asynchronously */ result |= ocf_metadata_segment_init( &ctrl->segment[i], cache, raw, - lock_page, unlock_page, + i == metadata_segment_collision, superblock); if (result) @@ -494,17 +467,9 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, ocf_cache_log(cache, log_info, "Metadata size on device: %llu kiB\n", ocf_metadata_data_offset(&cache->metadata) / KiB); - result = ocf_metadata_concurrency_attached_init(&cache->metadata.lock, + ocf_metadata_concurrency_attached_init(&cache->metadata.lock, cache, ctrl->raw_desc[metadata_segment_hash]. - layout->entries, - (uint32_t)ctrl->raw_desc[metadata_segment_collision]. - layout->pages); - if (result) { - ocf_cache_log(cache, log_err, "Failed to initialize attached " - "metadata concurrency\n"); - ocf_metadata_deinit_variable_size(cache); - return result; - } + layout->entries); return 0; } diff --git a/src/metadata/metadata_collision.c b/src/metadata/metadata_collision.c index a9c2baf34..3c3a7f403 100644 --- a/src/metadata/metadata_collision.c +++ b/src/metadata/metadata_collision.c @@ -273,7 +273,7 @@ void ocf_metadata_start_collision_shared_access(struct ocf_cache *cache, &ctrl->raw_desc[metadata_segment_collision]; uint32_t page = ocf_metadata_raw_page(raw, line); - ocf_collision_start_shared_access(&cache->metadata.lock, page); + ocf_metadata_raw_page_lock_modify(raw, page); } /* must be called under global metadata read(shared) lock */ @@ -286,7 +286,7 @@ void ocf_metadata_end_collision_shared_access(struct ocf_cache *cache, &ctrl->raw_desc[metadata_segment_collision]; uint32_t page = ocf_metadata_raw_page(raw, line); - ocf_collision_end_shared_access(&cache->metadata.lock, page); + ocf_metadata_raw_page_unlock_modify(raw, page); } diff --git a/src/metadata/metadata_passive_update.c b/src/metadata/metadata_passive_update.c index 7e6d1ad47..8102047da 100644 --- a/src/metadata/metadata_passive_update.c +++ b/src/metadata/metadata_passive_update.c @@ -15,7 +15,6 @@ #include "metadata_io.h" #include "metadata_raw.h" #include "metadata_segment.h" -#include "../concurrency/ocf_concurrency.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" #include "../utils/utils_cache_line.h" diff --git a/src/metadata/metadata_raw.c b/src/metadata/metadata_raw.c index db435dec5..134eb042d 100644 --- a/src/metadata/metadata_raw.c +++ b/src/metadata/metadata_raw.c @@ -98,9 +98,7 @@ static int _raw_ram_deinit(ocf_cache_t cache, /* * RAM Implementation - Initialize */ -static int _raw_ram_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +static int _raw_ram_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw) { size_t mem_pool_size; @@ -108,8 +106,7 @@ static int _raw_ram_init(ocf_cache_t cache, OCF_DEBUG_TRACE(cache); - /* TODO: caller should specify explicitly whether to init mio conc? */ - if (lock_page_pfn) { + if (flush_asynch) { ret = ocf_mio_concurrency_init(&raw->mio_conc, raw->layout->offset, raw->layout->pages, cache); if (ret) @@ -127,9 +124,6 @@ static int _raw_ram_init(ocf_cache_t cache, } ENV_BUG_ON(env_memset(raw->mem_pool, mem_pool_size, 0)); - raw->lock_page = lock_page_pfn; - raw->unlock_page = unlock_page_pfn; - return 0; } @@ -364,11 +358,9 @@ static int _raw_ram_flush_all_fill(ocf_cache_t cache, OCF_DEBUG_PARAM(cache, "Line = %u, Page = %u", line, raw_page); - if (raw->lock_page) - raw->lock_page(cache, raw, raw_page); + ocf_metadata_raw_page_lock_copy(raw, raw_page); ctx_data_wr_check(cache->owner, data, _RAW_RAM_ADDR(raw, line), size); - if (raw->unlock_page) - raw->unlock_page(cache, raw, raw_page); + ocf_metadata_raw_page_unlock_copy(raw, raw_page); ctx_data_zero_check(cache->owner, data, PAGE_SIZE - size); @@ -484,11 +476,9 @@ static int _raw_ram_flush_do_asynch_fill(ocf_cache_t cache, OCF_DEBUG_PARAM(cache, "Line = %u, Page = %u", line, raw_page); - if (raw->lock_page) - raw->lock_page(cache, raw, raw_page); + ocf_metadata_raw_page_lock_copy(raw, raw_page); ctx_data_wr_check(cache->owner, data, _RAW_RAM_ADDR(raw, line), size); - if (raw->unlock_page) - raw->unlock_page(cache, raw, raw_page); + ocf_metadata_raw_page_unlock_copy(raw, raw_page); ctx_data_zero_check(cache->owner, data, PAGE_SIZE - size); @@ -708,16 +698,72 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { * RAW Top interface implementation ******************************************************************************/ -int ocf_metadata_raw_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +static void _raw_page_locks_deinit(struct ocf_metadata_raw *raw) +{ + uint32_t i; + + if (!raw->page_locks) + return; + + for (i = 0; i < raw->layout->pages; i++) + env_rwsem_destroy(&raw->page_locks[i]); + + env_vfree(raw->page_locks); + raw->page_locks = NULL; +} + +static int _raw_page_locks_init(struct ocf_metadata_raw *raw) +{ + uint32_t i; + int result = 0; + + raw->page_locks = env_vzalloc(sizeof(*raw->page_locks) * + raw->layout->pages); + if (!raw->page_locks) + return -OCF_ERR_NO_MEM; + + for (i = 0; i < raw->layout->pages; i++) { + result = env_rwsem_init(&raw->page_locks[i]); + if (result) + break; + } + + if (result) { + while (i--) + env_rwsem_destroy(&raw->page_locks[i]); + env_vfree(raw->page_locks); + raw->page_locks = NULL; + } + + return result; +} + +int ocf_metadata_raw_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw) { + int result; + ENV_BUG_ON(raw->raw_type < metadata_raw_type_min); ENV_BUG_ON(raw->raw_type >= metadata_raw_type_max); raw->iface = &(IRAW[raw->raw_type]); - return raw->iface->init(cache, lock_page_pfn, unlock_page_pfn, raw); + raw->page_locks = NULL; + + result = raw->iface->init(cache, flush_asynch, raw); + if (result) + return result; + + /* Volatile RAW is never flushed, so it needs no page locks */ + if (flush_asynch && raw->raw_type != metadata_raw_type_volatile && + raw->layout->pages) { + result = _raw_page_locks_init(raw); + if (result) { + raw->iface->deinit(cache, raw); + raw->iface = NULL; + } + } + + return result; } int ocf_metadata_raw_deinit(ocf_cache_t cache, @@ -728,6 +774,8 @@ int ocf_metadata_raw_deinit(ocf_cache_t cache, if (!raw->iface) return 0; + _raw_page_locks_deinit(raw); + result = raw->iface->deinit(cache, raw); raw->iface = NULL; diff --git a/src/metadata/metadata_raw.h b/src/metadata/metadata_raw.h index deade2623..5f8936483 100644 --- a/src/metadata/metadata_raw.h +++ b/src/metadata/metadata_raw.h @@ -53,8 +53,6 @@ struct ocf_metadata_raw; /** * @brief Container page lock/unlock callback */ -typedef void (*ocf_flush_page_synch_t)(ocf_cache_t cache, - struct ocf_metadata_raw *raw, uint32_t page); /** * @brief RAW instance descriptor @@ -83,8 +81,7 @@ struct ocf_metadata_raw { void *priv; /*!< Private data - context */ - ocf_flush_page_synch_t lock_page; /*!< Page lock callback */ - ocf_flush_page_synch_t unlock_page; /*!< Page unlock callback */ + env_rwsem *page_locks; /*!< Page locks (NULL if not used) */ struct ocf_alock *mio_conc; }; @@ -93,9 +90,7 @@ struct ocf_metadata_raw { * RAW container interface */ struct raw_iface { - int (*init)(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + int (*init)(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw); int (*deinit)(ocf_cache_t cache, @@ -150,14 +145,12 @@ struct raw_iface { * @brief Initialize RAW instance * * @param cache - Cache instance - * @param lock_page_pfn - Optional page lock callback - * @param lock_page_pfn - Optional page unlock callback + * @param flush_asynch - RAW is flushed asynchronously (flush_do_asynch), + * concurrently with modifications of its content * @param raw - RAW descriptor * @return 0 - Operation success, otherwise error */ -int ocf_metadata_raw_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +int ocf_metadata_raw_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw); /** @@ -220,6 +213,62 @@ static inline uint32_t ocf_metadata_raw_page(struct ocf_metadata_raw* raw, return raw->iface->page(raw, entry); } +/** + * @brief Lock RAW page for modification of its content + * + * Multiple entries within the same page can be modified concurrently, + * as modifications of the particular entries are synchronized by + * higher level locks. + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_lock_modify( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_down_read(&raw->page_locks[page]); +} + +/** + * @brief Unlock RAW page locked for modification of its content + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_unlock_modify( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_up_read(&raw->page_locks[page]); +} + +/** + * @brief Lock RAW page for copying consistent snapshot of its content + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_lock_copy( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_down_write(&raw->page_locks[page]); +} + +/** + * @brief Unlock RAW page locked for copying + * + * @param raw - RAW descriptor + * @param page - Page number + */ +static inline void ocf_metadata_raw_page_unlock_copy( + struct ocf_metadata_raw *raw, uint32_t page) +{ + if (raw->page_locks) + env_rwsem_up_write(&raw->page_locks[page]); +} + /** * @brief Access specified element of metadata directly * diff --git a/src/metadata/metadata_raw_dynamic.c b/src/metadata/metadata_raw_dynamic.c index c3708ca2a..c05ddaab7 100644 --- a/src/metadata/metadata_raw_dynamic.c +++ b/src/metadata/metadata_raw_dynamic.c @@ -157,9 +157,7 @@ int raw_dynamic_deinit(ocf_cache_t cache, /* * RAM DYNAMIC Implementation - Initialize */ -int raw_dynamic_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +int raw_dynamic_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw) { struct _raw_ctrl *ctrl; @@ -172,8 +170,7 @@ int raw_dynamic_init(ocf_cache_t cache, if (raw->layout->entry_size > PAGE_SIZE) return -1; - /* TODO: caller should specify explicitly whether to init mio conc? */ - if (lock_page_pfn) { + if (flush_asynch) { ret = ocf_mio_concurrency_init(&raw->mio_conc, raw->layout->offset, raw->layout->pages, cache); if (ret) @@ -195,9 +192,6 @@ int raw_dynamic_init(ocf_cache_t cache, raw->priv = ctrl; - raw->lock_page = lock_page_pfn; - raw->unlock_page = unlock_page_pfn; - return 0; } @@ -532,12 +526,10 @@ static int raw_dynamic_flush_all_fill(ocf_cache_t cache, raw_page = page - context->ssd_pages_offset; OCF_DEBUG_PARAM(cache, "Page = %u", raw_page); - if (raw->lock_page) - raw->lock_page(cache, raw, raw_page); + ocf_metadata_raw_page_lock_copy(raw, raw_page); ctx_data_wr_check(cache->owner, data, _raw_dynamic_page_rd(raw, raw_page), PAGE_SIZE); - if (raw->unlock_page) - raw->unlock_page(cache, raw, raw_page); + ocf_metadata_raw_page_unlock_copy(raw, raw_page); return 0; } diff --git a/src/metadata/metadata_raw_dynamic.h b/src/metadata/metadata_raw_dynamic.h index 79a13fbd7..790bb8c57 100644 --- a/src/metadata/metadata_raw_dynamic.h +++ b/src/metadata/metadata_raw_dynamic.h @@ -15,9 +15,7 @@ /* * RAW DYNAMIC - Initialize */ -int raw_dynamic_init(ocf_cache_t cache, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, +int raw_dynamic_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw); /* diff --git a/src/metadata/metadata_segment.c b/src/metadata/metadata_segment.c index be66d693a..ef2cc49ba 100644 --- a/src/metadata/metadata_segment.c +++ b/src/metadata/metadata_segment.c @@ -11,13 +11,12 @@ int ocf_metadata_segment_init_in_place( struct ocf_metadata_segment *segment, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock) { int result; - result = ocf_metadata_raw_init(cache, lock_page_pfn, unlock_page_pfn, raw); + result = ocf_metadata_raw_init(cache, flush_asynch, raw); if (result) return result; @@ -32,8 +31,7 @@ int ocf_metadata_segment_init( struct ocf_metadata_segment **self, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock) { struct ocf_metadata_segment *segment; @@ -44,8 +42,7 @@ int ocf_metadata_segment_init( return -OCF_ERR_NO_MEM; result = ocf_metadata_segment_init_in_place(segment, - cache, raw, lock_page_pfn, unlock_page_pfn, - superblock); + cache, raw, flush_asynch, superblock); if (result) env_vfree(segment); diff --git a/src/metadata/metadata_segment.h b/src/metadata/metadata_segment.h index 84b69c3f1..915dca14f 100644 --- a/src/metadata/metadata_segment.h +++ b/src/metadata/metadata_segment.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2020-2021 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -21,16 +22,14 @@ int ocf_metadata_segment_init_in_place( struct ocf_metadata_segment *segment, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock); int ocf_metadata_segment_init( struct ocf_metadata_segment **self, struct ocf_cache *cache, struct ocf_metadata_raw *raw, - ocf_flush_page_synch_t lock_page_pfn, - ocf_flush_page_synch_t unlock_page_pfn, + bool flush_asynch, struct ocf_metadata_segment *superblock); void ocf_metadata_segment_destroy(struct ocf_cache *cache, diff --git a/src/metadata/metadata_status.h b/src/metadata/metadata_status.h index 271552618..169d0ebdd 100644 --- a/src/metadata/metadata_status.h +++ b/src/metadata/metadata_status.h @@ -8,7 +8,7 @@ #ifndef __METADATA_STATUS_H__ #define __METADATA_STATUS_H__ -#include "../concurrency/ocf_metadata_concurrency.h" +#include "../ocf_cache_priv.h" #include "metadata_cache_line.h" /******************************************************************************* diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index 0d00b0476..0d9f02b5c 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -33,10 +33,8 @@ struct ocf_metadata_lock /*!< global metadata lock (GML) */ env_spinlock lru[OCF_NUM_LRU_LISTS]; /*!< Fast locks for lru list */ env_spinlock partition[OCF_USER_IO_CLASS_MAX]; /* partition lock */ - env_rwsem *collision_pages; /*!< Collision table page locks */ ocf_cache_t cache; /*!< Parent cache object */ uint32_t num_hash_entries; /*!< Hash bucket count */ - uint32_t num_collision_pages; /*!< Collision table page count */ }; /** diff --git a/src/metadata/metadata_superblock.c b/src/metadata/metadata_superblock.c index 58b4ddbfb..ddfda7561 100644 --- a/src/metadata/metadata_superblock.c +++ b/src/metadata/metadata_superblock.c @@ -638,7 +638,7 @@ int ocf_metadata_superblock_init( return -OCF_ERR_NO_MEM; result = ocf_metadata_segment_init_in_place(&sb->segment, cache, - raw, NULL, NULL, &sb->segment); + raw, false, &sb->segment); if (result) { env_vfree(sb); diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 316edb628..55b39e72e 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -20,6 +20,7 @@ #include "../engine/engine_common.h" #include "../ocf_seq_cutoff.h" #include "../prefetch/ocf_prefetch_priv.h" +#include "../concurrency/ocf_metadata_concurrency.h" /* Close if opened */ void cache_mngt_core_deinit(ocf_core_t core) diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 6e93451fe..503dd6af1 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -21,6 +21,7 @@ #include "../utils/utils_pipeline.h" #include "../ocf_request.h" #include "../ocf_def_priv.h" +#include "../concurrency/ocf_metadata_concurrency.h" struct ocf_mngt_cache_flush_context; typedef void (*ocf_flush_complete_t)(struct ocf_mngt_cache_flush_context *, int); diff --git a/src/mngt/ocf_mngt_io_class.c b/src/mngt/ocf_mngt_io_class.c index b3a8cdff6..8046fe656 100644 --- a/src/mngt/ocf_mngt_io_class.c +++ b/src/mngt/ocf_mngt_io_class.c @@ -11,6 +11,7 @@ #include "../engine/cache_engine.h" #include "../utils/utils_user_part.h" #include "../ocf_lru.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "ocf_env.h" static uint64_t _ocf_mngt_count_user_parts_min_size(struct ocf_cache *cache) diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 349c5b801..093546cd5 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -14,6 +14,7 @@ #include "utils/utils_generator.h" #include "utils/utils_parallelize.h" #include "concurrency/ocf_concurrency.h" +#include "concurrency/ocf_metadata_concurrency.h" #include "mngt/ocf_mngt_common.h" #include "engine/engine_zero.h" #include "ocf_cache_priv.h" diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 82882feea..11549899e 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -9,6 +9,7 @@ #include "../engine/cache_engine.h" #include "../engine/engine_common.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_request.h" #include "utils_cleaner.h" #include "utils_user_part.h" From df363fe6bff56dcd5890fc907b198edd38b5143c Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 15:01:07 +0200 Subject: [PATCH 4/8] metadata: Make flush_asynch gate asynchronous flush Initialize synchronization primitives only for metadata segments that use flush_async and block asynchronous flush for all the other segments. Signed-off-by: Robert Baldyga --- src/metadata/metadata_raw.c | 166 +++++++++++++++++----------- src/metadata/metadata_raw.h | 17 ++- src/metadata/metadata_raw_dynamic.c | 22 +--- 3 files changed, 122 insertions(+), 83 deletions(-) diff --git a/src/metadata/metadata_raw.c b/src/metadata/metadata_raw.c index 134eb042d..916ddf08d 100644 --- a/src/metadata/metadata_raw.c +++ b/src/metadata/metadata_raw.c @@ -77,6 +77,49 @@ static bool _raw_ssd_page_is_valid(struct ocf_metadata_raw *raw, uint32_t page) +static void _raw_page_locks_deinit(struct ocf_metadata_raw *raw) +{ + uint32_t i; + + if (!raw->page_locks) + return; + + for (i = 0; i < raw->layout->pages; i++) + env_rwsem_destroy(&raw->page_locks[i]); + + env_vfree(raw->page_locks); + raw->page_locks = NULL; +} + +static int _raw_page_locks_init(struct ocf_metadata_raw *raw) +{ + uint32_t i; + int result = 0; + + if (!raw->layout->pages) + return 0; + + raw->page_locks = env_vzalloc(sizeof(*raw->page_locks) * + raw->layout->pages); + if (!raw->page_locks) + return -OCF_ERR_NO_MEM; + + for (i = 0; i < raw->layout->pages; i++) { + result = env_rwsem_init(&raw->page_locks[i]); + if (result) + break; + } + + if (result) { + while (i--) + env_rwsem_destroy(&raw->page_locks[i]); + env_vfree(raw->page_locks); + raw->page_locks = NULL; + } + + return result; +} + /* * RAM Implementation - De-Initialize */ @@ -90,35 +133,48 @@ static int _raw_ram_deinit(ocf_cache_t cache, raw->mem_pool = NULL; } + _raw_page_locks_deinit(raw); ocf_mio_concurrency_deinit(&raw->mio_conc); return 0; } /* - * RAM Implementation - Initialize + * RAM Implementation - Initialize common part + * + * @param mio_conc - serialize metadata IOs to the same pages + * @param page_locks - synchronize page copying with content modifications */ -static int _raw_ram_init(ocf_cache_t cache, bool flush_asynch, - struct ocf_metadata_raw *raw) +static int _raw_ram_init_common(ocf_cache_t cache, + struct ocf_metadata_raw *raw, bool mio_conc, bool page_locks) { size_t mem_pool_size; int ret; OCF_DEBUG_TRACE(cache); - if (flush_asynch) { + if (mio_conc) { ret = ocf_mio_concurrency_init(&raw->mio_conc, raw->layout->offset, raw->layout->pages, cache); if (ret) return ret; } + if (page_locks) { + ret = _raw_page_locks_init(raw); + if (ret) { + ocf_mio_concurrency_deinit(&raw->mio_conc); + return ret; + } + } + /* Allocate memory pool for entries */ mem_pool_size = raw->layout->pages; mem_pool_size *= PAGE_SIZE; raw->mem_pool_limit = mem_pool_size; raw->mem_pool = env_secure_alloc(mem_pool_size); if (!raw->mem_pool) { + _raw_page_locks_deinit(raw); ocf_mio_concurrency_deinit(&raw->mio_conc); return -OCF_ERR_NO_MEM; } @@ -127,6 +183,43 @@ static int _raw_ram_init(ocf_cache_t cache, bool flush_asynch, return 0; } +/* + * RAM Implementation - Initialize + * + * Asynchronous flush writes pages copied from memory concurrently with + * content modifications and with other flushes, so it requires page locks + * and serialization of IOs to the same pages. + */ +static int _raw_ram_init(ocf_cache_t cache, bool flush_asynch, + struct ocf_metadata_raw *raw) +{ + return _raw_ram_init_common(cache, raw, flush_asynch, flush_asynch); +} + +/* + * RAM VOLATILE Implementation - Initialize + * + * Volatile RAW is never written to the cache device. + */ +static int _raw_volatile_init(ocf_cache_t cache, bool flush_asynch, + struct ocf_metadata_raw *raw) +{ + return _raw_ram_init_common(cache, raw, false, false); +} + +/* + * RAM ATOMIC Implementation - Initialize + * + * Asynchronous flush doesn't write metadata pages, so no IO serialization + * is needed, but flush_all still copies pages concurrently with content + * modifications, which requires page locks. + */ +static int _raw_atomic_init(ocf_cache_t cache, bool flush_asynch, + struct ocf_metadata_raw *raw) +{ + return _raw_ram_init_common(cache, raw, false, flush_asynch); +} + /* * RAM Implementation - Size of */ @@ -661,7 +754,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .flush_do_asynch = raw_dynamic_flush_do_asynch, }, [metadata_raw_type_volatile] = { - .init = _raw_ram_init, + .init = _raw_volatile_init, .deinit = _raw_ram_deinit, .size_of = _raw_ram_size_of, .size_on_ssd = raw_volatile_size_on_ssd, @@ -677,7 +770,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .flush_do_asynch = raw_volatile_flush_do_asynch, }, [metadata_raw_type_atomic] = { - .init = _raw_ram_init, + .init = _raw_atomic_init, .deinit = _raw_ram_deinit, .size_of = _raw_ram_size_of, .size_on_ssd = _raw_ram_size_on_ssd, @@ -698,72 +791,17 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { * RAW Top interface implementation ******************************************************************************/ -static void _raw_page_locks_deinit(struct ocf_metadata_raw *raw) -{ - uint32_t i; - - if (!raw->page_locks) - return; - - for (i = 0; i < raw->layout->pages; i++) - env_rwsem_destroy(&raw->page_locks[i]); - - env_vfree(raw->page_locks); - raw->page_locks = NULL; -} - -static int _raw_page_locks_init(struct ocf_metadata_raw *raw) -{ - uint32_t i; - int result = 0; - - raw->page_locks = env_vzalloc(sizeof(*raw->page_locks) * - raw->layout->pages); - if (!raw->page_locks) - return -OCF_ERR_NO_MEM; - - for (i = 0; i < raw->layout->pages; i++) { - result = env_rwsem_init(&raw->page_locks[i]); - if (result) - break; - } - - if (result) { - while (i--) - env_rwsem_destroy(&raw->page_locks[i]); - env_vfree(raw->page_locks); - raw->page_locks = NULL; - } - - return result; -} - int ocf_metadata_raw_init(ocf_cache_t cache, bool flush_asynch, struct ocf_metadata_raw *raw) { - int result; - ENV_BUG_ON(raw->raw_type < metadata_raw_type_min); ENV_BUG_ON(raw->raw_type >= metadata_raw_type_max); raw->iface = &(IRAW[raw->raw_type]); + raw->flush_asynch = flush_asynch; raw->page_locks = NULL; - result = raw->iface->init(cache, flush_asynch, raw); - if (result) - return result; - - /* Volatile RAW is never flushed, so it needs no page locks */ - if (flush_asynch && raw->raw_type != metadata_raw_type_volatile && - raw->layout->pages) { - result = _raw_page_locks_init(raw); - if (result) { - raw->iface->deinit(cache, raw); - raw->iface = NULL; - } - } - - return result; + return raw->iface->init(cache, flush_asynch, raw); } int ocf_metadata_raw_deinit(ocf_cache_t cache, @@ -774,8 +812,6 @@ int ocf_metadata_raw_deinit(ocf_cache_t cache, if (!raw->iface) return 0; - _raw_page_locks_deinit(raw); - result = raw->iface->deinit(cache, raw); raw->iface = NULL; diff --git a/src/metadata/metadata_raw.h b/src/metadata/metadata_raw.h index 5f8936483..cebc4d3cc 100644 --- a/src/metadata/metadata_raw.h +++ b/src/metadata/metadata_raw.h @@ -81,6 +81,8 @@ struct ocf_metadata_raw { void *priv; /*!< Private data - context */ + bool flush_asynch; /*!< Asynchronous flush enabled */ + env_rwsem *page_locks; /*!< Page locks (NULL if not used) */ struct ocf_alock *mio_conc; @@ -145,8 +147,11 @@ struct raw_iface { * @brief Initialize RAW instance * * @param cache - Cache instance - * @param flush_asynch - RAW is flushed asynchronously (flush_do_asynch), - * concurrently with modifications of its content + * @param flush_asynch - Enable asynchronous flush (flush_mark and + * flush_do_asynch). RAW content is then modified at runtime, + * concurrently with any kind of flush, including flush_all, + * and RAW must assure consistency of flushed pages. + * Not supported by all RAW types. * @param raw - RAW descriptor * @return 0 - Operation success, otherwise error */ @@ -369,6 +374,9 @@ static inline void ocf_metadata_raw_flush_mark(ocf_cache_t cache, struct ocf_metadata_raw *raw, struct ocf_request *req, uint32_t map_idx, int to_state, uint8_t start, uint8_t stop) { + if (!raw->flush_asynch) + return; + raw->iface->flush_mark(cache, req, map_idx, to_state, start, stop); } @@ -376,6 +384,11 @@ static inline int ocf_metadata_raw_flush_do_asynch(ocf_cache_t cache, struct ocf_request *req, struct ocf_metadata_raw *raw, ocf_req_end_t complete) { + if (!raw->flush_asynch) { + complete(req, -OCF_ERR_NOT_SUPP); + return -OCF_ERR_NOT_SUPP; + } + return raw->iface->flush_do_asynch(cache, req, raw, complete); } diff --git a/src/metadata/metadata_raw_dynamic.c b/src/metadata/metadata_raw_dynamic.c index c05ddaab7..3ffeeabae 100644 --- a/src/metadata/metadata_raw_dynamic.c +++ b/src/metadata/metadata_raw_dynamic.c @@ -141,8 +141,6 @@ int raw_dynamic_deinit(ocf_cache_t cache, OCF_DEBUG_TRACE(cache); - ocf_mio_concurrency_deinit(&raw->mio_conc); - for (i = 0; i < raw->layout->pages; i++) env_secure_free(ctrl->pages[i], PAGE_SIZE); @@ -163,29 +161,23 @@ int raw_dynamic_init(ocf_cache_t cache, bool flush_asynch, struct _raw_ctrl *ctrl; size_t size = sizeof(*ctrl) + (sizeof(ctrl->pages[0]) * raw->layout->pages); - int ret; OCF_DEBUG_TRACE(cache); + /* Asynchronous flush is not supported */ + if (flush_asynch) + return -OCF_ERR_NOT_SUPP; + if (raw->layout->entry_size > PAGE_SIZE) return -1; - if (flush_asynch) { - ret = ocf_mio_concurrency_init(&raw->mio_conc, - raw->layout->offset, raw->layout->pages, cache); - if (ret) - return ret; - } ctrl = env_vmalloc(size); - if (!ctrl) { - ocf_mio_concurrency_deinit(&raw->mio_conc); + if (!ctrl) return -1; - } ENV_BUG_ON(env_memset(ctrl, size, 0)); if (env_mutex_init(&ctrl->lock)) { - ocf_mio_concurrency_deinit(&raw->mio_conc); env_vfree(ctrl); return -1; } @@ -526,10 +518,8 @@ static int raw_dynamic_flush_all_fill(ocf_cache_t cache, raw_page = page - context->ssd_pages_offset; OCF_DEBUG_PARAM(cache, "Page = %u", raw_page); - ocf_metadata_raw_page_lock_copy(raw, raw_page); ctx_data_wr_check(cache->owner, data, _raw_dynamic_page_rd(raw, raw_page), PAGE_SIZE); - ocf_metadata_raw_page_unlock_copy(raw, raw_page); return 0; } @@ -567,7 +557,7 @@ void raw_dynamic_flush_all(ocf_cache_t cache, struct ocf_metadata_raw *raw, context->ssd_pages_offset, raw->layout->pages, 0, raw_dynamic_flush_all_fill, raw_dynamic_flush_all_complete, - raw->mio_conc); + NULL); if (result) OCF_CMPL_RET(priv, result); } From 8546864a1fd376c77c7aebe7ab291f8dfb786dec Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 19:24:48 +0200 Subject: [PATCH 5/8] concurrency: Move metadata lock out of metadata structure The metadata lock protects cache mapping consistency, which is a concern of the concurrency layer, not of metadata itself. Move its definition to concurrency/ocf_metadata_lock.h and its instance from struct ocf_metadata to struct ocf_cache. Signed-off-by: Robert Baldyga --- src/cleaning/acp.c | 12 +++---- src/cleaning/alru.c | 14 ++++---- src/concurrency/ocf_metadata_concurrency.c | 22 ++++++------ src/concurrency/ocf_metadata_concurrency.h | 8 ++--- src/concurrency/ocf_metadata_lock.h | 31 ++++++++++++++++ src/engine/engine_wo.c | 6 ++-- src/metadata/metadata.c | 6 ++-- src/metadata/metadata_structs.h | 19 ---------- src/mngt/ocf_mngt_cache.c | 32 ++++++++--------- src/mngt/ocf_mngt_common.c | 12 +++---- src/mngt/ocf_mngt_flush.c | 36 +++++++++---------- src/mngt/ocf_mngt_io_class.c | 4 +-- src/ocf_cache_priv.h | 3 ++ src/ocf_lru.c | 20 +++++------ src/ocf_request.h | 1 + src/utils/utils_cleaner.c | 8 ++--- .../ocf_metadata_concurrency.c | 3 +- 17 files changed, 127 insertions(+), 110 deletions(-) create mode 100644 src/concurrency/ocf_metadata_lock.h diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index 9996d39d4..97b4e46a3 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -515,11 +515,11 @@ static int ocf_acp_update_handle(ocf_parallelize_t parallelize, begin = portion*shard_id; end = OCF_MIN(portion*(shard_id + 1), entries); - ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_start_shared_access(&cache->metadata_lock, lock_idx); for (hash = begin; hash < end; hash++) { OCF_COND_RESCHED_DEFAULT(step); - ocf_hb_id_naked_lock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_lock_rd(&cache->metadata_lock, hash); cline = ocf_metadata_get_hash(cache, hash); while (cline != terminator) { @@ -530,9 +530,9 @@ static int ocf_acp_update_handle(ocf_parallelize_t parallelize, cline = ocf_metadata_get_collision_next(cache, cline); } - ocf_hb_id_naked_unlock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_unlock_rd(&cache->metadata_lock, hash); } - ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_end_shared_access(&cache->metadata_lock, lock_idx); return 0; } @@ -638,7 +638,7 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, unsigned lock_idx = ocf_metadata_concurrency_next_idx( cache->cleaner.io_queue); - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, lock_idx, core_id, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, lock_idx, core_id, core_line); ocf_engine_lookup_map_entry(cache, &info, core_id, @@ -651,7 +651,7 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, info.coll_idx); } - ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, lock_idx, core_id, + ocf_hb_cline_prot_unlock_rd(&cache->metadata_lock, lock_idx, core_id, core_line); return locked ? info.coll_idx : diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index be8e40fd1..52498955c 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -657,11 +657,11 @@ static int ocf_alru_update_handle(ocf_parallelize_t parallelize, begin = portion*shard_id; end = OCF_MIN((uint64_t)portion*(shard_id + 1), entries); - ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_start_shared_access(&cache->metadata_lock, lock_idx); for (hash = begin; hash < end; hash++) { OCF_COND_RESCHED_DEFAULT(step); - ocf_hb_id_naked_lock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_lock_rd(&cache->metadata_lock, hash); cline = ocf_metadata_get_hash(cache, hash); while (cline != terminator) { @@ -672,9 +672,9 @@ static int ocf_alru_update_handle(ocf_parallelize_t parallelize, cline = ocf_metadata_get_collision_next(cache, cline); } - ocf_hb_id_naked_unlock_rd(&cache->metadata.lock, hash); + ocf_hb_id_naked_unlock_rd(&cache->metadata_lock, hash); } - ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_end_shared_access(&cache->metadata_lock, lock_idx); return 0; } @@ -1079,7 +1079,7 @@ static void alru_clean(struct alru_context *ctx) return; } - if (ocf_metadata_try_start_exclusive_access(&cache->metadata.lock)) { + if (ocf_metadata_try_start_exclusive_access(&cache->metadata_lock)) { alru_clean_complete(fctx, 0); return; } @@ -1098,7 +1098,7 @@ static void alru_clean(struct alru_context *ctx) ocf_cleaner_sort_flush_data(fctx->flush_data, to_clean); ocf_cleaner_do_flush_data_async(cache, fctx->flush_data, to_clean, &fctx->attribs); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return; } @@ -1107,7 +1107,7 @@ static void alru_clean(struct alru_context *ctx) env_ticks_to_secs(env_get_tick_count()); end: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); alru_clean_complete(fctx, 0); } diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c index 53c6fcd8b..da7d6114a 100644 --- a/src/concurrency/ocf_metadata_concurrency.c +++ b/src/concurrency/ocf_metadata_concurrency.c @@ -373,7 +373,7 @@ void ocf_hb_id_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock, } /* number of hash entries */ -#define _NUM_HASH_ENTRIES req->cache->metadata.lock.num_hash_entries +#define _NUM_HASH_ENTRIES req->cache->metadata_lock.num_hash_entries /* true if hashes are monotonic */ #define _IS_MONOTONIC(req) (req->map[0].hash + req->core_line_count <= \ @@ -448,10 +448,10 @@ void ocf_hb_req_prot_lock_rd(struct ocf_request *req) { ocf_cache_line_t hash; - ocf_metadata_start_shared_access(&req->cache->metadata.lock, + ocf_metadata_start_shared_access(&req->cache->metadata_lock, req->lock_idx); for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_lock(&req->cache->metadata_lock, hash, OCF_METADATA_RD); } } @@ -461,10 +461,10 @@ void ocf_hb_req_prot_unlock_rd(struct ocf_request *req) ocf_cache_line_t hash; for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_unlock(&req->cache->metadata_lock, hash, OCF_METADATA_RD); } - ocf_metadata_end_shared_access(&req->cache->metadata.lock, + ocf_metadata_end_shared_access(&req->cache->metadata_lock, req->lock_idx); } @@ -472,10 +472,10 @@ void ocf_hb_req_prot_lock_wr(struct ocf_request *req) { ocf_cache_line_t hash; - ocf_metadata_start_shared_access(&req->cache->metadata.lock, + ocf_metadata_start_shared_access(&req->cache->metadata_lock, req->lock_idx); for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_lock(&req->cache->metadata_lock, hash, OCF_METADATA_WR); } } @@ -485,11 +485,11 @@ void ocf_hb_req_prot_lock_upgrade(struct ocf_request *req) ocf_cache_line_t hash; for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_unlock(&req->cache->metadata_lock, hash, OCF_METADATA_RD); } for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_lock(&req->cache->metadata_lock, hash, OCF_METADATA_WR); } } @@ -499,9 +499,9 @@ void ocf_hb_req_prot_unlock_wr(struct ocf_request *req) ocf_cache_line_t hash; for_each_req_hash_asc(req, hash) { - ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash, + ocf_hb_id_naked_unlock(&req->cache->metadata_lock, hash, OCF_METADATA_WR); } - ocf_metadata_end_shared_access(&req->cache->metadata.lock, + ocf_metadata_end_shared_access(&req->cache->metadata_lock, req->lock_idx); } diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h index 3a9fb5cb1..71cffcbf3 100644 --- a/src/concurrency/ocf_metadata_concurrency.h +++ b/src/concurrency/ocf_metadata_concurrency.h @@ -58,18 +58,18 @@ static inline void ocf_metadata_lru_unlock_all( } #define OCF_METADATA_LRU_LOCK(ev_list) \ - ocf_metadata_lru_lock(&cache->metadata.lock, \ + ocf_metadata_lru_lock(&cache->metadata_lock, \ ev_list) #define OCF_METADATA_LRU_UNLOCK(ev_list) \ - ocf_metadata_lru_unlock(&cache->metadata.lock, \ + ocf_metadata_lru_unlock(&cache->metadata_lock, \ ev_list) #define OCF_METADATA_LRU_LOCK_ALL() \ - ocf_metadata_lru_lock_all(&cache->metadata.lock) + ocf_metadata_lru_lock_all(&cache->metadata_lock) #define OCF_METADATA_LRU_UNLOCK_ALL() \ - ocf_metadata_lru_unlock_all(&cache->metadata.lock) + ocf_metadata_lru_unlock_all(&cache->metadata_lock) static inline void ocf_metadata_partition_lock( struct ocf_metadata_lock *metadata_lock, diff --git a/src/concurrency/ocf_metadata_lock.h b/src/concurrency/ocf_metadata_lock.h new file mode 100644 index 000000000..cabbee9a6 --- /dev/null +++ b/src/concurrency/ocf_metadata_lock.h @@ -0,0 +1,31 @@ +/* + * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2025 Huawei Technologies + * Copyright(c) 2026 Unvertical + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __OCF_METADATA_LOCK_H__ +#define __OCF_METADATA_LOCK_H__ + +#include "ocf/ocf.h" +#include "ocf_env.h" +#include "../metadata/metadata_lru.h" + +#define OCF_METADATA_GLOBAL_LOCK_IDX_BITS 2 +#define OCF_NUM_GLOBAL_META_LOCKS (1 << (OCF_METADATA_GLOBAL_LOCK_IDX_BITS)) + +struct ocf_metadata_global_lock { + env_rwsem sem; +} __attribute__((aligned(64))); + +struct ocf_metadata_lock { + struct ocf_metadata_global_lock global[OCF_NUM_GLOBAL_META_LOCKS]; + /*!< global metadata lock (GML) */ + env_spinlock lru[OCF_NUM_LRU_LISTS]; /*!< Fast locks for lru list */ + env_spinlock partition[OCF_USER_IO_CLASS_MAX]; /* partition lock */ + ocf_cache_t cache; /*!< Parent cache object */ + uint32_t num_hash_entries; /*!< Hash bucket count */ +}; + +#endif /* __OCF_METADATA_LOCK_H__ */ diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index 50d4c635d..a73f9d9c4 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -73,7 +73,7 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) s = ocf_map_line_start_block(req, line); e = ocf_map_line_end_block(req, line); - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, req->lock_idx, entry->core_id, entry->core_line); @@ -117,7 +117,7 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) == valid); } - ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_unlock_rd(&cache->metadata_lock, req->lock_idx, entry->core_id, entry->core_line); @@ -137,7 +137,7 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) offset += increment; if (i <= e) { - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, req->lock_idx, entry->core_id, entry->core_line); } diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index ce696be06..47406da04 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -467,7 +467,7 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, ocf_cache_log(cache, log_info, "Metadata size on device: %llu kiB\n", ocf_metadata_data_offset(&cache->metadata) / KiB); - ocf_metadata_concurrency_attached_init(&cache->metadata.lock, + ocf_metadata_concurrency_attached_init(&cache->metadata_lock, cache, ctrl->raw_desc[metadata_segment_hash]. layout->entries); @@ -1260,7 +1260,7 @@ int ocf_metadata_init(struct ocf_cache *cache, if (ret) return ret; - ret = ocf_metadata_concurrency_init(&cache->metadata.lock); + ret = ocf_metadata_concurrency_init(&cache->metadata_lock); if (ret) { ocf_metadata_deinit_fixed_size(cache); return ret; @@ -1274,7 +1274,7 @@ void ocf_metadata_deinit(struct ocf_cache *cache) OCF_DEBUG_TRACE(cache); ocf_metadata_deinit_fixed_size(cache); - ocf_metadata_concurrency_deinit(&cache->metadata.lock); + ocf_metadata_concurrency_deinit(&cache->metadata_lock); } void ocf_metadata_error(struct ocf_cache *cache) diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index 0d9f02b5c..1631bf557 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -20,23 +20,6 @@ * @brief Metadata private structures */ -#define OCF_METADATA_GLOBAL_LOCK_IDX_BITS 2 -#define OCF_NUM_GLOBAL_META_LOCKS (1 << (OCF_METADATA_GLOBAL_LOCK_IDX_BITS)) - -struct ocf_metadata_global_lock { - env_rwsem sem; -} __attribute__((aligned(64))); - -struct ocf_metadata_lock -{ - struct ocf_metadata_global_lock global[OCF_NUM_GLOBAL_META_LOCKS]; - /*!< global metadata lock (GML) */ - env_spinlock lru[OCF_NUM_LRU_LISTS]; /*!< Fast locks for lru list */ - env_spinlock partition[OCF_USER_IO_CLASS_MAX]; /* partition lock */ - ocf_cache_t cache; /*!< Parent cache object */ - uint32_t num_hash_entries; /*!< Hash bucket count */ -}; - /** * @brief Metadata control structure */ @@ -58,8 +41,6 @@ struct ocf_metadata { uint64_t data_offset; /*!< Start of cache data area on cache device (in bytes) */ - - struct ocf_metadata_lock lock; }; /** diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index c765aa637..4c4470395 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -601,10 +601,10 @@ static void ocf_mngt_cline_rebuild_metadata(ocf_cache_t cache, hash_index = ocf_metadata_hash_func(cache, core_line, core_id); - ocf_hb_id_naked_lock_wr(&cache->metadata.lock, hash_index); + ocf_hb_id_naked_lock_wr(&cache->metadata_lock, hash_index); ocf_metadata_add_to_collision(cache, core_id, core_line, hash_index, cline); - ocf_hb_id_naked_unlock_wr(&cache->metadata.lock, hash_index); + ocf_hb_id_naked_unlock_wr(&cache->metadata_lock, hash_index); ocf_lru_init_cline(cache, cline); @@ -3705,11 +3705,11 @@ int ocf_mngt_cache_promotion_set_policy(ocf_cache_t cache, ocf_promotion_t type) if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_promotion_set_policy(cache->promotion_policy, type); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return result; } @@ -3721,11 +3721,11 @@ int ocf_mngt_cache_promotion_get_policy(ocf_cache_t cache, ocf_promotion_t *type if (ocf_cache_is_standby(cache)) return -OCF_ERR_CACHE_STANDBY; - ocf_metadata_start_shared_access(&cache->metadata.lock, 0); + ocf_metadata_start_shared_access(&cache->metadata_lock, 0); *type = cache->conf_meta->promotion_policy_type; - ocf_metadata_end_shared_access(&cache->metadata.lock, 0); + ocf_metadata_end_shared_access(&cache->metadata_lock, 0); return 0; } @@ -3738,11 +3738,11 @@ int ocf_mngt_cache_promotion_get_param(ocf_cache_t cache, ocf_promotion_t type, if (ocf_cache_is_standby(cache)) return -OCF_ERR_CACHE_STANDBY; - ocf_metadata_start_shared_access(&cache->metadata.lock, 0); + ocf_metadata_start_shared_access(&cache->metadata_lock, 0); result = ocf_promotion_get_param(cache, type, param_id, param_value); - ocf_metadata_end_shared_access(&cache->metadata.lock, 0); + ocf_metadata_end_shared_access(&cache->metadata_lock, 0); return result; } @@ -3758,11 +3758,11 @@ int ocf_mngt_cache_promotion_set_param(ocf_cache_t cache, ocf_promotion_t type, if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_promotion_set_param(cache, type, param_id, param_value); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return result; } @@ -3781,7 +3781,7 @@ int ocf_mngt_cache_prefetch_set_policy(ocf_cache_t cache, ocf_pf_mask_t mask) if (mask & ~valid_mask) return -OCF_ERR_INVAL; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); old_mask = cache->conf_meta->prefetch_mask; cache->conf_meta->prefetch_mask = mask; @@ -3802,7 +3802,7 @@ int ocf_mngt_cache_prefetch_set_policy(ocf_cache_t cache, ocf_pf_mask_t mask) } } - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return 0; } @@ -3814,11 +3814,11 @@ int ocf_mngt_cache_prefetch_get_policy(ocf_cache_t cache, ocf_pf_mask_t *mask) if (ocf_cache_is_standby(cache)) return -OCF_ERR_CACHE_STANDBY; - ocf_metadata_start_shared_access(&cache->metadata.lock, 0); + ocf_metadata_start_shared_access(&cache->metadata_lock, 0); *mask = cache->conf_meta->prefetch_mask; - ocf_metadata_end_shared_access(&cache->metadata.lock, 0); + ocf_metadata_end_shared_access(&cache->metadata_lock, 0); return 0; } @@ -3839,11 +3839,11 @@ int ocf_mngt_cache_prefetch_set_param(ocf_cache_t cache, ocf_pf_id_t pf_id, if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); ret = ocf_prefetch_set_param(cache, pf_id, param_id, param_value); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 55b39e72e..57da32d83 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -43,11 +43,11 @@ void cache_mngt_core_remove_from_cleaning_pol(ocf_core_t core) ocf_cache_t cache = ocf_core_get_cache(core); ocf_core_id_t core_id = ocf_core_get_id(core); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); ocf_cleaning_remove_core(cache, core_id); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); } /* Deinitialize core metadata in attached metadata */ @@ -66,7 +66,7 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) prev_cline = terminator; lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue); - ocf_hb_id_prot_lock_wr(&cache->metadata.lock, lock_idx, hash); + ocf_hb_id_prot_lock_wr(&cache->metadata_lock, lock_idx, hash); curr_cline = ocf_metadata_get_hash(cache, hash); while (curr_cline != terminator) { @@ -100,7 +100,7 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) else curr_cline = ocf_metadata_get_hash(cache, hash); } - ocf_hb_id_prot_unlock_wr(&cache->metadata.lock, lock_idx, hash); + ocf_hb_id_prot_unlock_wr(&cache->metadata_lock, lock_idx, hash); /* Check whether all the cachelines from the hash bucket were sparsed */ if (curr_cline == terminator) @@ -116,7 +116,7 @@ void cache_mngt_core_remove_from_meta(ocf_core_t core) ocf_cache_t cache = ocf_core_get_cache(core); ocf_core_id_t core_id = ocf_core_get_id(core); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); /* In metadata mark data this core was removed from cache */ core->conf_meta->valid = false; @@ -128,7 +128,7 @@ void cache_mngt_core_remove_from_meta(ocf_core_t core) env_bit_clear(core_id, cache->conf_meta->valid_core_bitmap); cache->conf_meta->core_count--; - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); } /* Deinit in-memory structures related to this core */ diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 503dd6af1..b0078bd60 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -186,7 +186,7 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, uint32_t line, dirty_found = 0, dirty_total = 0; unsigned ret = 0; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); dirty_total = env_atomic_read(&core->runtime_meta->dirty_clines); if (!dirty_total) { @@ -224,10 +224,10 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, if ((line + 1) % 131072 == 0) { ocf_metadata_end_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); env_cond_resched(); ocf_metadata_start_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); } } @@ -238,7 +238,7 @@ static int _ocf_mngt_get_blocks(ocf_cache_t cache, ocf_core_id_t core_id, *num = dirty_found; unlock: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } @@ -268,7 +268,7 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, return 0; } - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); core_count = cache->conf_meta->core_count; if (core_count == 0) @@ -343,10 +343,10 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, if ((line + 1) % 131072 == 0) { ocf_metadata_end_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); env_cond_resched(); ocf_metadata_start_exclusive_access( - &cache->metadata.lock); + &cache->metadata_lock); } } @@ -370,7 +370,7 @@ static int _ocf_mngt_get_flush_containers(ocf_cache_t cache, free_core_revmap: env_vfree(core_revmap); unlock: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } @@ -476,9 +476,9 @@ static int _ofc_flush_container_step(struct ocf_request *req) struct flush_container *fc = req->priv; ocf_cache_t cache = fc->cache; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); _ocf_mngt_flush_portion(fc); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return 0; } @@ -876,10 +876,10 @@ static void _ocf_mngt_cache_invalidate(ocf_pipeline_t pipeline, void *priv, ocf_cache_t cache = context->cache; int result; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_metadata_sparse_range(cache, context->purge.core_id, 0, context->purge.end_byte); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, result); } @@ -895,10 +895,10 @@ static void _ocf_mngt_cache_detach_cline_range(ocf_pipeline_t pipeline, free_detached_before = env_atomic_read(&cache->free_detached.runtime->curr_size); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_metadata_detach_cline_range(cache, context->begin, context->end); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); free_detached_after = env_atomic_read(&cache->free_detached.runtime->curr_size); @@ -922,9 +922,9 @@ int ocf_mngt_cache_attach_cline_range(ocf_cache_t cache, free_detached_before = env_atomic_read(&cache->free_detached.runtime->curr_size); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = ocf_metadata_restore_cline_range(cache, begin, end); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); free_detached_after = env_atomic_read(&cache->free_detached.runtime->curr_size); @@ -1345,11 +1345,11 @@ int ocf_mngt_cache_cleaning_set_param(ocf_cache_t cache, ocf_cleaning_t type, if (!ocf_cache_is_device_attached(cache)) return -OCF_ERR_CACHE_DETACHED; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); ret = ocf_cleaning_set_param(cache, type, param_id, param_value); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); return ret; } diff --git a/src/mngt/ocf_mngt_io_class.c b/src/mngt/ocf_mngt_io_class.c index 8046fe656..a356eb180 100644 --- a/src/mngt/ocf_mngt_io_class.c +++ b/src/mngt/ocf_mngt_io_class.c @@ -302,7 +302,7 @@ int ocf_mngt_cache_io_classes_configure(ocf_cache_t cache, if (!old_config) return -OCF_ERR_NO_MEM; - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + ocf_metadata_start_exclusive_access(&cache->metadata_lock); result = env_memcpy(old_config, sizeof(cache->user_parts), cache->user_parts, sizeof(cache->user_parts)); @@ -327,7 +327,7 @@ int ocf_mngt_cache_io_classes_configure(ocf_cache_t cache, } out_cpy: - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + ocf_metadata_end_exclusive_access(&cache->metadata_lock); env_free(old_config); return result; diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 9ed18a7bf..dae764b66 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -14,6 +14,7 @@ #include "ocf_core_priv.h" #include "ocf_part.h" #include "metadata/metadata_structs.h" +#include "concurrency/ocf_metadata_lock.h" #include "utils/utils_list.h" #include "utils/utils_pipeline.h" #include "utils/utils_async_lock.h" @@ -68,6 +69,8 @@ struct ocf_cache { struct ocf_metadata metadata; + struct ocf_metadata_lock metadata_lock; + struct { /* cache get/put counter */ struct env_refcnt cache; diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 093546cd5..27aadebc3 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -411,7 +411,7 @@ static inline bool _lru_trylock_hash(struct ocf_lru_iter *iter, } return ocf_hb_cline_naked_trylock_wr( - &iter->cache->metadata.lock, + &iter->cache->metadata_lock, core_id, core_line); } @@ -424,7 +424,7 @@ static inline void _lru_unlock_hash(struct ocf_lru_iter *iter, } ocf_hb_cline_naked_unlock_wr( - &iter->cache->metadata.lock, + &iter->cache->metadata_lock, core_id, core_line); } @@ -487,7 +487,7 @@ static inline ocf_cache_line_t lru_req_next_cline(struct ocf_request *req, uint64_t t_core_line; ocf_part_id_t tmp_part_id; - ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_lock(&cache->metadata_lock, curr_lru); tmp_part_id = ocf_metadata_get_partition_id(cache, cline); if (tmp_part_id == PARTITION_FREELIST) @@ -533,7 +533,7 @@ static inline ocf_cache_line_t lru_req_next_cline(struct ocf_request *req, if (ret == end_marker) ocf_cache_line_unlock_wr(c, cline); lru_wr_unlock: - ocf_metadata_lru_unlock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_unlock(&cache->metadata_lock, curr_lru); return ret; } @@ -560,7 +560,7 @@ static inline ocf_cache_line_t lru_iter_eviction_next(struct ocf_lru_iter *iter, lru_iter_advance(iter); curr_lru = iter->lru_idx; - ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_lock(&cache->metadata_lock, curr_lru); list = ocf_lru_get_list(part, curr_lru, iter->clean); @@ -579,7 +579,7 @@ static inline ocf_cache_line_t lru_iter_eviction_next(struct ocf_lru_iter *iter, } } - ocf_metadata_lru_unlock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_unlock(&cache->metadata_lock, curr_lru); if (cline == end_marker && !_lru_lru_is_empty(iter)) { /* mark list as empty */ @@ -613,7 +613,7 @@ static inline ocf_cache_line_t lru_iter_free_next(struct ocf_lru_iter *iter, lru_iter_advance(iter); curr_lru = iter->lru_idx; - ocf_metadata_lru_lock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_lock(&cache->metadata_lock, curr_lru); list = ocf_lru_get_list(free, curr_lru, true); @@ -627,7 +627,7 @@ static inline ocf_cache_line_t lru_iter_free_next(struct ocf_lru_iter *iter, ocf_lru_repart_locked(cache, cline, free, dst_part); } - ocf_metadata_lru_unlock(&cache->metadata.lock, curr_lru); + ocf_metadata_lru_unlock(&cache->metadata_lock, curr_lru); if (cline == end_marker && !_lru_lru_is_empty(iter)) { /* mark list as empty */ @@ -723,7 +723,7 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, lru_idx = io_queue->lru_idx++ % OCF_NUM_LRU_LISTS; lock_idx = ocf_metadata_concurrency_next_idx(io_queue); - ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_start_shared_access(&cache->metadata_lock, lock_idx); OCF_METADATA_LRU_LOCK_ALL(); @@ -739,7 +739,7 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, OCF_METADATA_LRU_UNLOCK_ALL(); - ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx); + ocf_metadata_end_shared_access(&cache->metadata_lock, lock_idx); if (i == 0) { env_atomic_set(&ctx->cleaner_running, 0); diff --git a/src/ocf_request.h b/src/ocf_request.h index aadb6b918..bb46564c6 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -12,6 +12,7 @@ #include "ocf_io_priv.h" #include "ocf_def_priv.h" #include "metadata/metadata_structs.h" +#include "concurrency/ocf_metadata_lock.h" typedef enum { /* modes inherited from user API */ diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 11549899e..c2f798703 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -336,7 +336,7 @@ static int _ocf_cleaner_update_metadata(struct ocf_request *req) cache_line = iter->coll_idx; - ocf_hb_cline_prot_lock_wr(&cache->metadata.lock, + ocf_hb_cline_prot_lock_wr(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); @@ -352,7 +352,7 @@ static int _ocf_cleaner_update_metadata(struct ocf_request *req) cache_line); } - ocf_hb_cline_prot_unlock_wr(&cache->metadata.lock, + ocf_hb_cline_prot_unlock_wr(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); } @@ -511,13 +511,13 @@ static int _ocf_cleaner_fire_core(struct ocf_request *req) if (!iter->flush) continue; - ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_lock_rd(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); _ocf_cleaner_core_submit_io(req, iter); - ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, + ocf_hb_cline_prot_unlock_rd(&cache->metadata_lock, req->lock_idx, req->map[i].core_id, req->map[i].core_line); } diff --git a/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c b/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c index 30badaca5..4e5cafdf6 100644 --- a/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c +++ b/tests/unit/tests/concurrency/ocf_metadata_concurrency.c/ocf_metadata_concurrency.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2019-2022 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -116,7 +117,7 @@ static void ocf_hb_req_prot_lock_rd_test01(void **state) const unsigned test_case_count = sizeof(test_cases) / sizeof(test_cases[0]); unsigned i; - req->cache->metadata.lock.num_hash_entries = 5; + req->cache->metadata_lock.num_hash_entries = 5; print_test_description("Verify hash locking order\n"); From 6f274a251115b8ceb4d451d3224cf15d8407b76b Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 19:44:39 +0200 Subject: [PATCH 6/8] concurrency: Move metadata lock lifecycle out of metadata The metadata lock lives as long as the cache, so initialize it together with other cache locks in mngt, and do the attach-time part in ocf_concurrency_init(). Metadata no longer initializes or uses the metadata lock. Signed-off-by: Robert Baldyga --- src/concurrency/ocf_concurrency.c | 4 ++++ src/metadata/metadata.c | 20 +------------------- src/mngt/ocf_mngt_cache.c | 13 +++++++++++++ 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/concurrency/ocf_concurrency.c b/src/concurrency/ocf_concurrency.c index b38172455..547b6cdc8 100644 --- a/src/concurrency/ocf_concurrency.c +++ b/src/concurrency/ocf_concurrency.c @@ -5,12 +5,16 @@ */ #include "ocf_concurrency.h" +#include "ocf_metadata_concurrency.h" #include "../metadata/metadata.h" int ocf_concurrency_init(struct ocf_cache *cache) { int result = 0; + ocf_metadata_concurrency_attached_init(&cache->metadata_lock, cache, + ocf_metadata_hash_entries(&cache->metadata)); + result = ocf_cache_line_concurrency_init( &cache->device->concurrency.cache_line, ocf_metadata_line_count(&cache->metadata), diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 47406da04..475a20805 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -17,7 +17,6 @@ #include "metadata_raw.h" #include "metadata_segment.h" #include "../concurrency/ocf_concurrency.h" -#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" #include "../utils/utils_cache_line.h" @@ -467,10 +466,6 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, ocf_cache_log(cache, log_info, "Metadata size on device: %llu kiB\n", ocf_metadata_data_offset(&cache->metadata) / KiB); - ocf_metadata_concurrency_attached_init(&cache->metadata_lock, - cache, ctrl->raw_desc[metadata_segment_hash]. - layout->entries); - return 0; } @@ -1252,21 +1247,9 @@ bool ocf_metadata_check(struct ocf_cache *cache, ocf_cache_line_t line) int ocf_metadata_init(struct ocf_cache *cache, ocf_cache_line_size_t cache_line_size) { - int ret; - OCF_DEBUG_TRACE(cache); - ret = ocf_metadata_init_fixed_size(cache, cache_line_size); - if (ret) - return ret; - - ret = ocf_metadata_concurrency_init(&cache->metadata_lock); - if (ret) { - ocf_metadata_deinit_fixed_size(cache); - return ret; - } - - return 0; + return ocf_metadata_init_fixed_size(cache, cache_line_size); } void ocf_metadata_deinit(struct ocf_cache *cache) @@ -1274,7 +1257,6 @@ void ocf_metadata_deinit(struct ocf_cache *cache) OCF_DEBUG_TRACE(cache); ocf_metadata_deinit_fixed_size(cache); - ocf_metadata_concurrency_deinit(&cache->metadata_lock); } void ocf_metadata_error(struct ocf_cache *cache) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 4c4470395..95585c2c6 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -957,6 +957,14 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, goto mutex_err; } + result = ocf_metadata_concurrency_init(&cache->metadata_lock); + if (result) { + ocf_log(params->ctx, log_err, + "Failed to allocate cache %s metadata lock\n", + new_cache_name); + goto metadata_lock_err; + } + result = !env_refcnt_inc(&cache->refcnt.cache); if (result) { ocf_log(params->ctx, log_crit, @@ -979,6 +987,8 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, return 0; cache_refcnt_inc_err: + ocf_metadata_concurrency_deinit(&cache->metadata_lock); +metadata_lock_err: env_spinlock_destroy(&cache->io_queues_lock); mutex_err: env_mutex_destroy(&cache->flush_mutex); @@ -1570,6 +1580,8 @@ static void _ocf_mngt_init_handle_error(ocf_ctx_t ctx, if (!params->flags.cache_alloc) return; + ocf_metadata_concurrency_deinit(&cache->metadata_lock); + env_spinlock_destroy(&cache->io_queues_lock); env_mutex_destroy(&cache->flush_mutex); @@ -2381,6 +2393,7 @@ static void _ocf_mngt_cache_dealloc(void *priv) ctx = cache->owner; ocf_metadata_deinit(cache); + ocf_metadata_concurrency_deinit(&cache->metadata_lock); if (!cache->metadata.is_volatile) ocf_metadata_io_close(ctx); From f03fcdc2ca96420c483c9a846f3642ec38f5c320 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 19:50:29 +0200 Subject: [PATCH 7/8] metadata: Clean up unneeded dependencies from the header Also rename metadata_structs.h to metadata_priv.h, as the role of the header changed recently. Signed-off-by: Robert Baldyga --- src/cleaning/cleaning_ops.h | 2 +- .../{metadata_structs.h => metadata_priv.h} | 14 +++++--------- src/ocf_cache_priv.h | 2 +- src/ocf_request.h | 2 +- 4 files changed, 8 insertions(+), 12 deletions(-) rename src/metadata/{metadata_structs.h => metadata_priv.h} (86%) diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index bb8b2274e..3cbf55ab3 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -12,7 +12,7 @@ #include "nop.h" #include "acp.h" #include "../metadata/metadata_superblock.h" -#include "../metadata/metadata_structs.h" +#include "../metadata/metadata_priv.h" #include "../ocf_cache_priv.h" #include "ocf_env_refcnt.h" diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_priv.h similarity index 86% rename from src/metadata/metadata_structs.h rename to src/metadata/metadata_priv.h index 1631bf557..a177c4f9c 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_priv.h @@ -5,15 +5,11 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef __METADATA_STRUCTS_H__ -#define __METADATA_STRUCTS_H__ - -#include "metadata_common.h" -#include "../ocf_space.h" -#include "../cleaning/cleaning.h" -#include "../ocf_request.h" -#include "metadata_superblock.h" +#ifndef __METADATA_PRIV_H__ +#define __METADATA_PRIV_H__ +#include "ocf/ocf.h" +#include "ocf_env.h" /** * @file metadata_priv.h @@ -79,4 +75,4 @@ static inline uint64_t ocf_metadata_data_offset( return metadata->data_offset; } -#endif /* __METADATA_STRUCTS_H__ */ +#endif /* __METADATA_PRIV_H__ */ diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index dae764b66..ac0c86344 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -13,7 +13,7 @@ #include "ocf_volume_priv.h" #include "ocf_core_priv.h" #include "ocf_part.h" -#include "metadata/metadata_structs.h" +#include "metadata/metadata_priv.h" #include "concurrency/ocf_metadata_lock.h" #include "utils/utils_list.h" #include "utils/utils_pipeline.h" diff --git a/src/ocf_request.h b/src/ocf_request.h index bb46564c6..867378d7d 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -11,7 +11,7 @@ #include "ocf_env.h" #include "ocf_io_priv.h" #include "ocf_def_priv.h" -#include "metadata/metadata_structs.h" +#include "metadata/metadata_priv.h" #include "concurrency/ocf_metadata_lock.h" typedef enum { From 0477530e6d5e354df86bd009fae1d5112444c471 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 7 Oct 2026 23:53:41 +0200 Subject: [PATCH 8/8] metadata: Unused includes cleanup Signed-off-by: Robert Baldyga --- src/metadata/metadata.c | 3 +-- src/metadata/metadata.h | 1 - src/metadata/metadata_collision.c | 1 - src/metadata/metadata_core.c | 1 - src/metadata/metadata_io.c | 5 ----- src/metadata/metadata_partition.c | 1 - src/metadata/metadata_passive_update.c | 5 ----- src/metadata/metadata_raw_atomic.c | 1 - src/metadata/metadata_raw_dynamic.c | 3 --- 9 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 475a20805..3cb584df9 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -16,11 +16,10 @@ #include "metadata_io.h" #include "metadata_raw.h" #include "metadata_segment.h" -#include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_cache_line_concurrency.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" #include "../utils/utils_cache_line.h" -#include "../utils/utils_io.h" #include "../utils/utils_pipeline.h" #include "../utils/utils_parallelize.h" diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index 6a376a148..f8939034e 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -9,7 +9,6 @@ #define __METADATA_H__ #include "metadata_common.h" -#include "../ocf_ctx_priv.h" #include "metadata_cleaning_policy.h" #include "metadata_lru.h" #include "metadata_partition.h" diff --git a/src/metadata/metadata_collision.c b/src/metadata/metadata_collision.c index 3c3a7f403..e0b5918d7 100644 --- a/src/metadata/metadata_collision.c +++ b/src/metadata/metadata_collision.c @@ -8,7 +8,6 @@ #include "ocf/ocf.h" #include "metadata.h" #include "metadata_internal.h" -#include "../utils/utils_cache_line.h" static inline void ocf_metadata_list_info_set( struct ocf_metadata_list_info *info, diff --git a/src/metadata/metadata_core.c b/src/metadata/metadata_core.c index 600b2f712..415690e35 100644 --- a/src/metadata/metadata_core.c +++ b/src/metadata/metadata_core.c @@ -5,7 +5,6 @@ */ #include "ocf/ocf.h" -#include "../ocf_priv.h" #include "metadata.h" #include "metadata_core.h" #include "metadata_internal.h" diff --git a/src/metadata/metadata_io.c b/src/metadata/metadata_io.c index 6fd629140..09e002652 100644 --- a/src/metadata/metadata_io.c +++ b/src/metadata/metadata_io.c @@ -7,11 +7,6 @@ #include "metadata.h" #include "metadata_io.h" #include "../ocf_priv.h" -#include "../engine/cache_engine.h" -#include "../engine/engine_common.h" -#include "../engine/engine_bf.h" -#include "../utils/utils_cache_line.h" -#include "../utils/utils_io.h" #include "../ocf_request.h" #include "../ocf_def_priv.h" #include "../concurrency/ocf_mio_concurrency.h" diff --git a/src/metadata/metadata_partition.c b/src/metadata/metadata_partition.c index b56d29ccb..252c97667 100644 --- a/src/metadata/metadata_partition.c +++ b/src/metadata/metadata_partition.c @@ -8,7 +8,6 @@ #include "ocf/ocf.h" #include "metadata.h" #include "metadata_internal.h" -#include "../utils/utils_user_part.h" ocf_part_id_t ocf_metadata_get_partition_id(struct ocf_cache *cache, ocf_cache_line_t line) diff --git a/src/metadata/metadata_passive_update.c b/src/metadata/metadata_passive_update.c index 8102047da..fbc9110ff 100644 --- a/src/metadata/metadata_passive_update.c +++ b/src/metadata/metadata_passive_update.c @@ -16,12 +16,7 @@ #include "metadata_raw.h" #include "metadata_segment.h" #include "../ocf_def_priv.h" -#include "../ocf_priv.h" -#include "../utils/utils_cache_line.h" -#include "../utils/utils_io.h" -#include "../utils/utils_pipeline.h" #include "../concurrency/ocf_pio_concurrency.h" -#include "../engine/engine_common.h" #define MAX_PASSIVE_IO_SIZE (32*MiB) diff --git a/src/metadata/metadata_raw_atomic.c b/src/metadata/metadata_raw_atomic.c index e41b8554e..257b053c7 100644 --- a/src/metadata/metadata_raw_atomic.c +++ b/src/metadata/metadata_raw_atomic.c @@ -10,7 +10,6 @@ #include "metadata_segment_id.h" #include "metadata_raw.h" #include "metadata_raw_atomic.h" -#include "../utils/utils_io.h" #include "../utils/utils_cache_line.h" #include "../ocf_def_priv.h" diff --git a/src/metadata/metadata_raw_dynamic.c b/src/metadata/metadata_raw_dynamic.c index 3ffeeabae..224b2d554 100644 --- a/src/metadata/metadata_raw_dynamic.c +++ b/src/metadata/metadata_raw_dynamic.c @@ -11,9 +11,6 @@ #include "metadata_raw_dynamic.h" #include "metadata_layout.h" #include "metadata_io.h" -#include "../engine/cache_engine.h" -#include "../engine/engine_common.h" -#include "../utils/utils_io.h" #include "../ocf_request.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h"