Add new putExternal type

Signed-off-by: JaySon-Huang <tshent@qq.com>
pingcap · Feb 11, 2022 · 6617289 · 6617289
1 parent 679c70a
commit 6617289
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 7 deletions.
diff --git a/dbms/src/Storages/Page/V2/PageFile.cpp b/dbms/src/Storages/Page/V2/PageFile.cpp
@@ -3,22 +3,22 @@
 #include <Common/FailPoint.h>
 #include <Common/ProfileEvents.h>
 #include <Common/StringUtils/StringUtils.h>
+#include <IO/WriteBufferFromFile.h>
 #include <IO/WriteHelpers.h>
+#include <Poco/File.h>
+#include <Storages/Page/PageUtil.h>
+#include <Storages/Page/V2/PageFile.h>
+#include <Storages/Page/WriteBatch.h>
 #include <common/logger_useful.h>
 
 #include <boost/algorithm/string/classification.hpp>
 #include <boost/algorithm/string/split.hpp>
+#include <ext/scope_guard.h>
 
 #ifndef __APPLE__
 #include <fcntl.h>
 #endif
 
-#include <IO/WriteBufferFromFile.h>
-#include <Poco/File.h>
-#include <Storages/Page/PageUtil.h>
-#include <Storages/Page/V2/PageFile.h>
-
-#include <ext/scope_guard.h>
 
 namespace CurrentMetrics
 {
@@ -86,6 +86,7 @@ std::pair<ByteBuffer, ByteBuffer> genWriteData( //
         switch (write.type)
         {
         case WriteBatch::WriteType::PUT:
+        case WriteBatch::WriteType::PUT_EXTERNAL:
         case WriteBatch::WriteType::UPSERT:
             if (write.read_buffer)
                 data_write_bytes += write.size;
@@ -119,9 +120,15 @@ std::pair<ByteBuffer, ByteBuffer> genWriteData( //
     PageOffset page_data_file_off = page_file.getDataFileAppendPos();
     for (auto & write : wb.getWrites())
     {
+        if (write.type == WriteBatch::WriteType::PUT_EXTERNAL)
+            write.type = WriteBatch::WriteType::PUT;
+
         PageUtil::put(meta_pos, static_cast<IsPut>(write.type));
         switch (write.type)
         {
+        case WriteBatch::WriteType::PUT_EXTERNAL:
+            throw Exception("Should not run into here with PUT_EXTERNAL");
+            break;
         case WriteBatch::WriteType::PUT:
         case WriteBatch::WriteType::UPSERT:
         {
@@ -340,6 +347,9 @@ void PageFile::LinkingMetaAdapter::linkToNewSequenceNext(WriteBatch::SequenceID
         const auto write_type = static_cast<WriteBatch::WriteType>(PageUtil::get<PageMetaFormat::IsPut>(pos));
         switch (write_type)
         {
+        case WriteBatch::WriteType::PUT_EXTERNAL:
+            throw Exception("Should not run into here with PUT_EXTERNAL");
+            break;
         case WriteBatch::WriteType::PUT:
         case WriteBatch::WriteType::UPSERT:
         {
@@ -564,6 +574,9 @@ void PageFile::MetaMergingReader::moveNext(PageFormat::Version * v)
         const auto write_type = static_cast<WriteBatch::WriteType>(PageUtil::get<PageMetaFormat::IsPut>(pos));
         switch (write_type)
         {
+        case WriteBatch::WriteType::PUT_EXTERNAL:
+            throw Exception("Should not run into here with PUT_EXTERNAL");
+            break;
         case WriteBatch::WriteType::PUT:
         case WriteBatch::WriteType::UPSERT:
         {

diff --git a/dbms/src/Storages/Page/V2/VersionSet/PageEntriesBuilder.cpp b/dbms/src/Storages/Page/V2/VersionSet/PageEntriesBuilder.cpp
@@ -8,6 +8,7 @@ void PageEntriesBuilder::apply(const PageEntriesEdit & edit)
     {
         switch (rec.type)
         {
+        case WriteBatch::WriteType::PUT_EXTERNAL:
         case WriteBatch::WriteType::PUT:
             current_version->put(rec.page_id, rec.entry);
             break;

diff --git a/dbms/src/Storages/Page/V2/VersionSet/PageEntriesVersionSetWithDelta.cpp b/dbms/src/Storages/Page/V2/VersionSet/PageEntriesVersionSetWithDelta.cpp
@@ -400,6 +400,7 @@ void DeltaVersionEditAcceptor::apply(PageEntriesEdit & edit)
     {
         switch (rec.type)
         {
+        case WriteBatch::WriteType::PUT_EXTERNAL:
         case WriteBatch::WriteType::PUT:
             this->applyPut(rec);
             break;
@@ -531,6 +532,7 @@ void DeltaVersionEditAcceptor::applyInplace(const String & name,
     {
         switch (rec.type)
         {
+        case WriteBatch::WriteType::PUT_EXTERNAL:
         case WriteBatch::WriteType::PUT:
             current->put(rec.page_id, rec.entry);
             break;

diff --git a/dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp b/dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp
@@ -8,6 +8,7 @@
 #include <Poco/Timer.h>
 #include <Storages/Page/V2/PageStorage.h>
 #include <Storages/Page/V2/gc/DataCompactor.h>
+#include <Storages/Page/WriteBatch.h>
 #include <Storages/PathPool.h>
 #include <TestUtils/MockDiskDelegator.h>
 
@@ -244,6 +245,7 @@ void dump_all_entries(PageFileSet & page_files, int32_t mode)
                 printf("%s\tseq: %9llu\t", page_file.toString().c_str(), sequence);
                 switch (record.type)
                 {
+                case DB::WriteBatch::WriteType::PUT_EXTERNAL:
                 case DB::WriteBatch::WriteType::PUT:
                     printf("PUT");
                     printPageEntry(record.page_id, record.entry);

diff --git a/dbms/src/Storages/Page/WriteBatch.h b/dbms/src/Storages/Page/WriteBatch.h
@@ -26,6 +26,8 @@ class WriteBatch : private boost::noncopyable
         // Create or update a Page. Now only used by GC.
         // Compare to `PUT`, this type won't create the RefPage{id} -> Page{id} by default.
         UPSERT = 3,
+        //
+        PUT_EXTERNAL = 4,
     };
 
     using SequenceID = UInt64;
@@ -86,7 +88,7 @@ class WriteBatch : private boost::noncopyable
     void putExternal(PageId page_id, UInt64 tag)
     {
         // External page's data is not managed by PageStorage, which means data is empty.
-        Write w{WriteType::PUT, page_id, tag, nullptr, 0, 0, {}, 0, 0, {}};
+        Write w{WriteType::PUT_EXTERNAL, page_id, tag, nullptr, 0, 0, {}, 0, 0, {}};
         writes.emplace_back(std::move(w));
     }