Skip to content

Commit

Permalink
Create dedicated cache FS
Browse files Browse the repository at this point in the history
  • Loading branch information
Y-- committed Jan 23, 2025
1 parent 0100cfe commit 5c4788d
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 1 deletion.
3 changes: 2 additions & 1 deletion third_party/cached_httpfs/http_file_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ shared_ptr<CachedFile> HTTPFileCache::GetCachedFile(const string &cache_dir, con
if (it != cached_files.end()) {
return it->second;
}
auto cache_entry = make_shared_ptr<CachedFile>(cache_dir, db->GetFileSystem(), key, cache_file);

auto cache_entry = make_shared_ptr<CachedFile>(cache_dir, cache_fs, key, cache_file);
if (cache_entry->Initialized() || cache_file) {
cached_files[key] = cache_entry;
return cache_entry;
Expand Down
11 changes: 11 additions & 0 deletions third_party/cached_httpfs/include/http_file_cache.hpp
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
#pragma once

#include "duckdb/main/client_data.hpp"
#include "duckdb/common/local_file_system.hpp"

namespace duckdb {

class CachedFileHandle;
class LocalFileSystem;

//! Represents a file that is intended to be fully downloaded, then used in parallel by multiple threads
class CachedFile : public enable_shared_from_this<CachedFile> {
Expand Down Expand Up @@ -78,6 +80,13 @@ class CachedFileHandle {
shared_ptr<CachedFile> file;
};

class LocalCacheFileSystem: public LocalFileSystem {
// TODO: we could lock down the LocalFileSystem to only allow path that are in the cache directory
std::string GetName() const override {
return "LocalCacheFileSystem";
}
};

class HTTPFileCache : public ClientContextState {
public:
explicit HTTPFileCache(ClientContext &context) {
Expand All @@ -88,6 +97,8 @@ class HTTPFileCache : public ClientContextState {
shared_ptr<CachedFile> GetCachedFile(const string &cache_dir, const string &key, bool create_cache);

private:
LocalCacheFileSystem cache_fs;

//! Database Instance
shared_ptr<DatabaseInstance> db;
//! Mutex to lock when getting the cached file (Parallel Only)
Expand Down

0 comments on commit 5c4788d

Please sign in to comment.