Skip to content

Commit

Permalink
[improve](file cache) rename the var QueryContext to QueryFileCacheCo…
Browse files Browse the repository at this point in the history
…ntext (apache#16272)
  • Loading branch information
Lchangliang authored Feb 1, 2023
1 parent ba026b6 commit 1c7c6b2
Show file tree
Hide file tree
Showing 6 changed files with 36 additions and 31 deletions.
23 changes: 13 additions & 10 deletions be/src/io/cache/block/block_file_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ std::string IFileCache::get_path_in_local_cache(const Key& key) const {
return fs::path(_cache_base_path) / key_str;
}

IFileCache::QueryContextHolderPtr IFileCache::get_query_context_holder(const TUniqueId& query_id) {
IFileCache::QueryFileCacheContextHolderPtr IFileCache::get_query_context_holder(
const TUniqueId& query_id) {
std::lock_guard cache_lock(_mutex);

if (!_enable_file_cache_query_limit) {
Expand All @@ -73,11 +74,11 @@ IFileCache::QueryContextHolderPtr IFileCache::get_query_context_holder(const TUn
/// if enable_filesystem_query_cache_limit is true,
/// we create context query for current query.
auto context = get_or_set_query_context(query_id, cache_lock);
return std::make_unique<QueryContextHolder>(query_id, this, context);
return std::make_unique<QueryFileCacheContextHolder>(query_id, this, context);
}

IFileCache::QueryContextPtr IFileCache::get_query_context(const TUniqueId& query_id,
std::lock_guard<std::mutex>& cache_lock) {
IFileCache::QueryFileCacheContextPtr IFileCache::get_query_context(
const TUniqueId& query_id, std::lock_guard<std::mutex>& cache_lock) {
auto query_iter = _query_map.find(query_id);
return (query_iter == _query_map.end()) ? nullptr : query_iter->second;
}
Expand All @@ -91,7 +92,7 @@ void IFileCache::remove_query_context(const TUniqueId& query_id) {
}
}

IFileCache::QueryContextPtr IFileCache::get_or_set_query_context(
IFileCache::QueryFileCacheContextPtr IFileCache::get_or_set_query_context(
const TUniqueId& query_id, std::lock_guard<std::mutex>& cache_lock) {
if (query_id.lo == 0 && query_id.hi == 0) {
return nullptr;
Expand All @@ -102,21 +103,23 @@ IFileCache::QueryContextPtr IFileCache::get_or_set_query_context(
return context;
}

auto query_context = std::make_shared<QueryContext>(_max_query_cache_size);
auto query_context = std::make_shared<QueryFileCacheContext>(_max_query_cache_size);
auto query_iter = _query_map.emplace(query_id, query_context).first;
return query_iter->second;
}

void IFileCache::QueryContext::remove(const Key& key, size_t offset, bool is_presistent,
size_t size, std::lock_guard<std::mutex>& cache_lock) {
void IFileCache::QueryFileCacheContext::remove(const Key& key, size_t offset, bool is_presistent,
size_t size,
std::lock_guard<std::mutex>& cache_lock) {
auto record = records.find({key, offset, is_presistent});
DCHECK(record != records.end());
lru_queue.remove(record->second, cache_lock);
records.erase({key, offset, is_presistent});
}

void IFileCache::QueryContext::reserve(const Key& key, size_t offset, bool is_presistent,
size_t size, std::lock_guard<std::mutex>& cache_lock) {
void IFileCache::QueryFileCacheContext::reserve(const Key& key, size_t offset, bool is_presistent,
size_t size,
std::lock_guard<std::mutex>& cache_lock) {
auto queue_iter = lru_queue.add(key, offset, is_presistent, size, cache_lock);
records.insert({{key, offset, is_presistent}, queue_iter});
}
Expand Down
34 changes: 18 additions & 16 deletions be/src/io/cache/block/block_file_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -178,13 +178,13 @@ class IFileCache {

/// Used to track and control the cache access of each query.
/// Through it, we can realize the processing of different queries by the cache layer.
struct QueryContext {
struct QueryFileCacheContext {
LRUQueue lru_queue;
AccessRecord records;

size_t max_cache_size = 0;

QueryContext(size_t max_cache_size) : max_cache_size(max_cache_size) {}
QueryFileCacheContext(size_t max_cache_size) : max_cache_size(max_cache_size) {}

void remove(const Key& key, size_t offset, bool is_presistent, size_t size,
std::lock_guard<std::mutex>& cache_lock);
Expand All @@ -201,31 +201,33 @@ class IFileCache {
LRUQueue& queue() { return lru_queue; }
};

using QueryContextPtr = std::shared_ptr<QueryContext>;
using QueryContextMap = std::unordered_map<TUniqueId, QueryContextPtr>;
using QueryFileCacheContextPtr = std::shared_ptr<QueryFileCacheContext>;
using QueryFileCacheContextMap = std::unordered_map<TUniqueId, QueryFileCacheContextPtr>;

QueryContextMap _query_map;
QueryFileCacheContextMap _query_map;

bool _enable_file_cache_query_limit = config::enable_file_cache_query_limit;

QueryContextPtr get_query_context(const TUniqueId& query_id, std::lock_guard<std::mutex>&);
QueryFileCacheContextPtr get_query_context(const TUniqueId& query_id,
std::lock_guard<std::mutex>&);

void remove_query_context(const TUniqueId& query_id);

QueryContextPtr get_or_set_query_context(const TUniqueId& query_id,
std::lock_guard<std::mutex>&);
QueryFileCacheContextPtr get_or_set_query_context(const TUniqueId& query_id,
std::lock_guard<std::mutex>&);

public:
/// Save a query context information, and adopt different cache policies
/// for different queries through the context cache layer.
struct QueryContextHolder {
QueryContextHolder(const TUniqueId& query_id, IFileCache* cache, QueryContextPtr context)
struct QueryFileCacheContextHolder {
QueryFileCacheContextHolder(const TUniqueId& query_id, IFileCache* cache,
QueryFileCacheContextPtr context)
: query_id(query_id), cache(cache), context(context) {}

QueryContextHolder& operator=(const QueryContextHolder&) = delete;
QueryContextHolder(const QueryContextHolder&) = delete;
QueryFileCacheContextHolder& operator=(const QueryFileCacheContextHolder&) = delete;
QueryFileCacheContextHolder(const QueryFileCacheContextHolder&) = delete;

~QueryContextHolder() {
~QueryFileCacheContextHolder() {
/// If only the query_map and the current holder hold the context_query,
/// the query has been completed and the query_context is released.
if (context) {
Expand All @@ -236,10 +238,10 @@ class IFileCache {

const TUniqueId& query_id;
IFileCache* cache = nullptr;
QueryContextPtr context;
QueryFileCacheContextPtr context;
};
using QueryContextHolderPtr = std::unique_ptr<QueryContextHolder>;
QueryContextHolderPtr get_query_context_holder(const TUniqueId& query_id);
using QueryFileCacheContextHolderPtr = std::unique_ptr<QueryFileCacheContextHolder>;
QueryFileCacheContextHolderPtr get_query_context_holder(const TUniqueId& query_id);
};

using CloudFileCachePtr = IFileCache*;
Expand Down
4 changes: 2 additions & 2 deletions be/src/io/cache/block/block_file_cache_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ CloudFileCachePtr FileCacheFactory::get_disposable_cache(const IFileCache::Key&
return _disposable_cache[KeyHash()(key) % _caches.size()].get();
}

std::vector<IFileCache::QueryContextHolderPtr> FileCacheFactory::get_query_context_holders(
std::vector<IFileCache::QueryFileCacheContextHolderPtr> FileCacheFactory::get_query_context_holders(
const TUniqueId& query_id) {
std::vector<IFileCache::QueryContextHolderPtr> holders;
std::vector<IFileCache::QueryFileCacheContextHolderPtr> holders;
for (const auto& cache : _caches) {
holders.push_back(cache->get_query_context_holder(query_id));
}
Expand Down
2 changes: 1 addition & 1 deletion be/src/io/cache/block/block_file_cache_factory.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class FileCacheFactory {

CloudFileCachePtr get_by_path(const IFileCache::Key& key);
CloudFileCachePtr get_disposable_cache(const IFileCache::Key& key);
std::vector<IFileCache::QueryContextHolderPtr> get_query_context_holders(
std::vector<IFileCache::QueryFileCacheContextHolderPtr> get_query_context_holders(
const TUniqueId& query_id);
FileCacheFactory() = default;
FileCacheFactory& operator=(const FileCacheFactory&) = delete;
Expand Down
2 changes: 1 addition & 1 deletion be/src/io/cache/block/block_lru_file_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ bool LRUFileCache::try_reserve(const Key& key, const TUniqueId& query_id, bool i
return true;
}

bool LRUFileCache::try_reserve_for_main_list(const Key& key, QueryContextPtr query_context,
bool LRUFileCache::try_reserve_for_main_list(const Key& key, QueryFileCacheContextPtr query_context,
bool is_persistent, size_t offset, size_t size,
std::lock_guard<std::mutex>& cache_lock) {
LRUQueue* queue = is_persistent ? &_persistent_queue : &_queue;
Expand Down
2 changes: 1 addition & 1 deletion be/src/io/cache/block/block_lru_file_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ class LRUFileCache final : public IFileCache {
bool try_reserve(const Key& key, const TUniqueId& query_id, bool is_persistent, size_t offset,
size_t size, std::lock_guard<std::mutex>& cache_lock) override;

bool try_reserve_for_main_list(const Key& key, QueryContextPtr query_context,
bool try_reserve_for_main_list(const Key& key, QueryFileCacheContextPtr query_context,
bool is_persistent, size_t offset, size_t size,
std::lock_guard<std::mutex>& cache_lock);

Expand Down

0 comments on commit 1c7c6b2

Please sign in to comment.