From 938bf07fc4dcdf511fd6b5c1164717cb50486d97 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 30 Jan 2023 15:27:54 +0800 Subject: [PATCH] remove tiflash-proxy for tiflash_compute mode (#6588) close pingcap/tiflash#6666 Signed-off-by: ywqzzy <592838129@qq.com> --- dbms/src/Flash/DiagnosticsService.cpp | 6 ++++++ dbms/src/Interpreters/AsynchronousMetrics.cpp | 2 ++ dbms/src/Server/Server.cpp | 14 +++++++++++--- dbms/src/Storages/Transaction/LearnerRead.cpp | 1 + dbms/src/Storages/Transaction/TMTContext.cpp | 9 ++++++++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/dbms/src/Flash/DiagnosticsService.cpp b/dbms/src/Flash/DiagnosticsService.cpp index d327a5ada5c..2087ac9515d 100644 --- a/dbms/src/Flash/DiagnosticsService.cpp +++ b/dbms/src/Flash/DiagnosticsService.cpp @@ -38,6 +38,12 @@ ::grpc::Status DiagnosticsService::server_info( ::diagnosticspb::ServerInfoResponse * response) try { + if (context.isDisaggregatedComputeMode()) + { + String err_msg = "tiflash compute node should be managed by AutoScaler instead of PD, this grpc should not be called be AutoScaler for now"; + LOG_ERROR(log, err_msg); + return ::grpc::Status(::grpc::StatusCode::INTERNAL, err_msg); + } const TiFlashRaftProxyHelper * helper = context.getTMTContext().getKVStore()->getProxyHelper(); if (helper) { diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index d1b074e9141..7a2d37829fb 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -121,6 +121,7 @@ static void calculateMaxAndSum(Max & max, Sum & sum, T x) FileUsageStatistics AsynchronousMetrics::getPageStorageFileUsage() { + RUNTIME_ASSERT(!context.isDisaggregatedComputeMode()); // Get from RegionPersister auto & tmt = context.getTMTContext(); auto & kvstore = tmt.getKVStore(); @@ -195,6 +196,7 @@ void AsynchronousMetrics::update() set("MaxDTBackgroundTasksLength", max_dt_background_tasks_length); } + if (!context.isDisaggregatedComputeMode()) { const FileUsageStatistics usage = getPageStorageFileUsage(); set("BlobFileNums", usage.total_file_num); diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 50c7140077a..5108c815931 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -248,6 +248,10 @@ struct TiFlashProxyConfig explicit TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config) { + auto disaggregated_mode = getDisaggregatedMode(config); + // tiflash_compute doesn't need proxy. + if (disaggregated_mode == DisaggregatedMode::Compute) + return; if (!config.has(config_prefix)) return; @@ -268,7 +272,6 @@ struct TiFlashProxyConfig else args_map[engine_store_advertise_address] = args_map[engine_store_address]; - auto disaggregated_mode = getDisaggregatedMode(config); args_map[engine_label] = getProxyLabelByDisaggregatedMode(disaggregated_mode); for (auto && [k, v] : args_map) @@ -843,10 +846,10 @@ int Server::main(const std::vector & /*args*/) RaftStoreProxyRunner proxy_runner(RaftStoreProxyRunner::RunRaftStoreProxyParms{&helper, proxy_conf}, log); - proxy_runner.run(); - if (proxy_conf.is_proxy_runnable) { + proxy_runner.run(); + LOG_INFO(log, "wait for tiflash proxy initializing"); while (!tiflash_instance_wrap.proxy_helper) std::this_thread::sleep_for(std::chrono::milliseconds(200)); @@ -1377,6 +1380,11 @@ int Server::main(const std::vector & /*args*/) WaitCheckRegionReady(tmt_context, *kvstore_ptr, terminate_signals_counter); } SCOPE_EXIT({ + if (!proxy_conf.is_proxy_runnable) + { + tmt_context.setStatusTerminated(); + return; + } if (proxy_conf.is_proxy_runnable && tiflash_instance_wrap.status != EngineStoreServerStatus::Running) { LOG_ERROR(log, "Current status of engine-store is NOT Running, should not happen"); diff --git a/dbms/src/Storages/Transaction/LearnerRead.cpp b/dbms/src/Storages/Transaction/LearnerRead.cpp index 8a4cdefa04b..7bd21b67be2 100644 --- a/dbms/src/Storages/Transaction/LearnerRead.cpp +++ b/dbms/src/Storages/Transaction/LearnerRead.cpp @@ -165,6 +165,7 @@ LearnerReadSnapshot doLearnerRead( const LoggerPtr & log) { assert(log != nullptr); + RUNTIME_ASSERT(!context.isDisaggregatedComputeMode()); auto & tmt = context.getTMTContext(); diff --git a/dbms/src/Storages/Transaction/TMTContext.cpp b/dbms/src/Storages/Transaction/TMTContext.cpp index bde42468495..f3897ec8827 100644 --- a/dbms/src/Storages/Transaction/TMTContext.cpp +++ b/dbms/src/Storages/Transaction/TMTContext.cpp @@ -67,7 +67,7 @@ static SchemaSyncerPtr createSchemaSyncer(bool exist_pd_addr, bool for_unit_test TMTContext::TMTContext(Context & context_, const TiFlashRaftConfig & raft_config, const pingcap::ClusterConfig & cluster_config) : context(context_) - , kvstore(std::make_shared(context, raft_config.snapshot_apply_method)) + , kvstore(context_.isDisaggregatedComputeMode() ? nullptr : std::make_shared(context, raft_config.snapshot_apply_method)) , region_table(context) , background_service(nullptr) , gc_manager(context) @@ -96,6 +96,10 @@ void TMTContext::updateSecurityConfig(const TiFlashRaftConfig & raft_config, con void TMTContext::restore(PathPool & path_pool, const TiFlashRaftProxyHelper * proxy_helper) { + // For tiflash_compute mode, kvstore should be nullptr, no need to restore region_table. + if (context.isDisaggregatedComputeMode()) + return; + kvstore->restore(path_pool, proxy_helper); region_table.restore(); store_status = StoreStatus::Ready; @@ -200,6 +204,9 @@ const std::unordered_set & TMTContext::getIgnoreDatabases() const void TMTContext::reloadConfig(const Poco::Util::AbstractConfiguration & config) { + if (context.isDisaggregatedComputeMode()) + return; + static constexpr const char * COMPACT_LOG_MIN_PERIOD = "flash.compact_log_min_period"; static constexpr const char * COMPACT_LOG_MIN_ROWS = "flash.compact_log_min_rows"; static constexpr const char * COMPACT_LOG_MIN_BYTES = "flash.compact_log_min_bytes";