Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix blobstore truncate unexpected normal files #3984

Merged
merged 11 commits into from
Feb 10, 2022
Merged

Fix blobstore truncate unexpected normal files #3984

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
3ef6795
Fix blobstore used removed file
jiaqizho Feb 9, 2022
1b34f18
fix tidy
jiaqizho Feb 9, 2022
6124f76
fix stress test
jiaqizho Feb 9, 2022
d766245
fix
jiaqizho Feb 9, 2022
cb87741
Fix BlobStore normal blob been truncate.
jiaqizho Feb 9, 2022
f7a9dfa
update
jiaqizho Feb 9, 2022
fdda4d3
Merge branch 'master' into fix-blobstore-normal-file-been-truncate
JaySon-Huang Feb 9, 2022
4ac6ad5
update
jiaqizho Feb 10, 2022
28b4e58
remove unused include
jiaqizho Feb 10, 2022
6a9fa32
update
jiaqizho Feb 10, 2022
ebd645d
Merge branch 'master' into fix-blobstore-normal-file-been-truncate
ti-chi-bot Feb 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions dbms/src/Storages/Page/V3/BlobFile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,16 @@ extern const char exception_before_page_file_write_sync[];
namespace PS::V3
{
BlobFile::BlobFile(String path_,
FileProviderPtr file_provider_,
bool truncate_if_exists)
FileProviderPtr file_provider_)
: file_provider{file_provider_}
, path(path_)
{
// TODO: support encryption file
wrfile = file_provider->newWriteReadableFile(
getPath(),
getEncryptionPath(),
truncate_if_exists,
/*create_new_encryption_info_*/ truncate_if_exists);
false,
/*create_new_encryption_info_*/ false);
}

void BlobFile::read(char * buffer, size_t offset, size_t size, const ReadLimiterPtr & read_limiter)
Expand Down
3 changes: 1 addition & 2 deletions dbms/src/Storages/Page/V3/BlobFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@ class BlobFile
{
public:
BlobFile(String path_,
FileProviderPtr file_provider_,
bool truncate_if_exists = true);
FileProviderPtr file_provider_);

~BlobFile();

Expand Down
11 changes: 9 additions & 2 deletions dbms/src/Storages/Page/V3/BlobStore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

#include <ext/scope_guard.h>
#include <mutex>

namespace ProfileEvents
{
extern const Event PSMWritePages;
Expand Down Expand Up @@ -475,7 +474,15 @@ std::vector<BlobFileId> BlobStore::getGCStats()
auto lock = stat->lock();
auto right_margin = stat->smap->getRightMargin();

// Avoid divide by zero
if (right_margin == 0)
jiaqizho marked this conversation as resolved.
Show resolved Hide resolved
{
LOG_FMT_TRACE(log, "Current blob is empty [blob_id={}, total size(all invalid)={}].", stat->id, stat->sm_total_size);
continue;
}

stat->sm_valid_rate = stat->sm_valid_size * 1.0 / right_margin;

if (stat->sm_valid_rate > 1.0)
{
LOG_FMT_ERROR(
jiaqizho marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -493,7 +500,6 @@ std::vector<BlobFileId> BlobStore::getGCStats()
if (stat->sm_valid_rate <= config.heavy_gc_valid_rate)
{
LOG_FMT_TRACE(log, "Current [blob_id={}] valid rate is {:.2f}, Need do compact GC", stat->id, stat->sm_valid_rate);
stat->sm_total_size = stat->sm_valid_size;
blob_need_gc.emplace_back(stat->id);

// Change current stat to read only
Expand All @@ -507,6 +513,7 @@ std::vector<BlobFileId> BlobStore::getGCStats()
if (right_margin != stat->sm_total_size)
{
auto blobfile = getBlobFile(stat->id);
LOG_FMT_TRACE(log, "Truncate blob file [blob_id={}] [origin size={}] [truncated size={}]", stat->id, stat->sm_total_size, right_margin);
blobfile->truncate(right_margin);
stat->sm_total_size = right_margin;
}
Expand Down