From 0485f5fa13af987ea9a186ca594ef575777a53a2 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 30 Jul 2024 10:36:40 +0800 Subject: [PATCH] 1 --- .../src/main/java/org/apache/doris/common/Config.java | 4 ++++ .../java/org/apache/doris/datasource/ExternalCatalog.java | 4 +--- .../java/org/apache/doris/datasource/ExternalDatabase.java | 2 +- .../org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 2 +- .../datasource/hudi/source/HudiCachedPartitionProcessor.java | 2 +- .../apache/doris/datasource/iceberg/IcebergMetadataCache.java | 4 ++-- .../java/org/apache/doris/datasource/metacache/MetaCache.java | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index ff7078b7e3a0ddf..506944b84a4148c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1986,6 +1986,10 @@ public class Config extends ConfigBase { "Max cache number of hive table name list."}) public static long max_hive_table_cache_num = 1000; + @ConfField(mutable = false, masterOnly = false, description = {"Hive表名缓存的最大数量。", + "Max cache number of hive table name list."}) + public static long max_meta_object_cache_num = 1000; + @ConfField(mutable = false, masterOnly = false, description = { "Hive分区表缓存的最大数量", "Max cache number of hive partition table" }) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 4f587ab74659584..2dfcec1d8aef4d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -238,7 +238,7 @@ public final synchronized void makeSureInitialized() { name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_hive_table_cache_num, + Config.max_meta_object_cache_num, ignored -> getFilteredDatabaseNames(), dbName -> Optional.ofNullable( buildDbForInit(dbName, Util.genIdByName(name, dbName), logType)), @@ -660,8 +660,6 @@ protected ExternalDatabase buildDbForInit(String dbName return new IcebergExternalDatabase(this, dbId, dbName); case MAX_COMPUTE: return new MaxComputeExternalDatabase(this, dbId, dbName); - //case HUDI: - //return new HudiExternalDatabase(this, dbId, dbName); case LAKESOUL: return new LakeSoulExternalDatabase(this, dbId, dbName); case TEST: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 2919633858f90cd..13db68eced2ed3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -145,7 +145,7 @@ public final synchronized void makeSureInitialized() { name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_hive_table_cache_num, + Config.max_meta_object_cache_num, ignored -> listTableNames(), tableName -> Optional.ofNullable( buildTableForInit(tableName, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index da88a03f2ebabe4..631362a5b417019 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -177,7 +177,7 @@ public void setNewFileCache() { CacheFactory fileCacheFactory = new CacheFactory( OptionalLong.of(fileMetaCacheTtlSecond >= HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE - ? fileMetaCacheTtlSecond : 86400L), + ? fileMetaCacheTtlSecond : 28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_external_file_cache_num, false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java index d9c1c2082717671..37a9a69743f984b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java @@ -52,7 +52,7 @@ public HudiCachedPartitionProcessor(long catalogId, ExecutorService executor) { this.catalogId = catalogId; this.executor = executor; CacheFactory partitionCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_hive_table_cache_num, false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java index dc11a6cacc24cf4..934451ea74eff36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java @@ -55,7 +55,7 @@ public class IcebergMetadataCache { public IcebergMetadataCache(ExecutorService executor) { CacheFactory snapshotListCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_hive_table_cache_num, false, @@ -63,7 +63,7 @@ public IcebergMetadataCache(ExecutorService executor) { this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor); CacheFactory tableCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_hive_table_cache_num, false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index c251db3a5c19f79..e3ad8668fb55258 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -57,7 +57,7 @@ public MetaCache(String name, CacheFactory namesCacheFactory = new CacheFactory( expireAfterWriteSec, refreshAfterWriteSec, - maxSize, + 1, // names cache has one and only one entry true, null); CacheFactory objCacheFactory = new CacheFactory(