From da7fe7d334d77025b172d404d94589a8777e9865 Mon Sep 17 00:00:00 2001 From: Sahith Kurapati Date: Tue, 24 Dec 2024 12:08:20 +0000 Subject: [PATCH] [PLAT-14335][PLAT-16308] Collect Connection Manager logs in support bundle Summary: This diff fixes 2 tickets: 1. [PLAT-14335] Collect Connection Manager logs in support bundle 2. [PLAT-16308] Support connection manager log rollover on YBA Logs folder before this change: ``` [yugabyte@yb-admin-sahith-uni-3-n1 ~]$ ls -ls ./tserver/logs/ total 16536 12 -rw-------. 1 yugabyte yugabyte 10311 Dec 23 20:30 postgresql-2024-12-23_202310.log 4096 -rw-------. 1 yugabyte yugabyte 6803 Dec 23 20:34 postgresql-2024-12-23_203102.log 0 lrwxrwxrwx. 1 yugabyte yugabyte 76 Dec 23 20:24 yb-tserver.ERROR -> yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.ERROR.20241223-202408.34213 0 lrwxrwxrwx. 1 yugabyte yugabyte 75 Dec 23 20:31 yb-tserver.INFO -> yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.INFO.20241223-203101.45710 0 lrwxrwxrwx. 1 yugabyte yugabyte 78 Dec 23 20:31 yb-tserver.WARNING -> yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.WARNING.20241223-203101.45710 4 -rw-r--r--. 1 yugabyte yugabyte 758 Dec 23 20:24 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.ERROR.20241223-202408.34213 120 -rw-r--r--. 1 yugabyte yugabyte 119667 Dec 23 20:29 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.INFO.20241223-202309.34213 4096 -rw-r--r--. 1 yugabyte yugabyte 147264 Dec 23 20:34 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.INFO.20241223-203101.45710 4 -rw-r--r--. 1 yugabyte yugabyte 1430 Dec 23 20:24 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.WARNING.20241223-202310.34213 4096 -rw-r--r--. 1 yugabyte yugabyte 5267 Dec 23 20:34 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.WARNING.20241223-203101.45710 4 -rw-r--r--. 1 yugabyte yugabyte 454 Dec 23 20:23 ysql-conn-mgr-2024-12-23_202310.log.34420 4 -rw-r--r--. 1 yugabyte yugabyte 208 Dec 23 20:30 ysql-conn-mgr-2024-12-23_203042.log.34420 4 -rw-r--r--. 1 yugabyte yugabyte 454 Dec 23 20:31 ysql-conn-mgr-2024-12-23_203102.log.45786 4096 -rw-r--r--. 1 yugabyte yugabyte 152 Dec 23 20:34 ysql-conn-mgr-2024-12-23_203434.log.45786 ``` Logs folder after this diff: ``` [yugabyte@yb-admin-sahith-uni-3-n1 ~]$ ls -ls ./tserver/logs/ total 16424 4 -rw-------. 1 yugabyte yugabyte 2023 Dec 23 20:30 postgresql-2024-12-23_202310.log.gz 4096 -rw-------. 1 yugabyte yugabyte 11336 Dec 23 20:35 postgresql-2024-12-23_203102.log 0 lrwxrwxrwx. 1 yugabyte yugabyte 76 Dec 23 20:24 yb-tserver.ERROR -> yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.ERROR.20241223-202408.34213 0 lrwxrwxrwx. 1 yugabyte yugabyte 75 Dec 23 20:31 yb-tserver.INFO -> yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.INFO.20241223-203101.45710 0 lrwxrwxrwx. 1 yugabyte yugabyte 78 Dec 23 20:31 yb-tserver.WARNING -> yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.WARNING.20241223-203101.45710 4 -rw-r--r--. 1 yugabyte yugabyte 758 Dec 23 20:24 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.ERROR.20241223-202408.34213 16 -rw-r--r--. 1 yugabyte yugabyte 13954 Dec 23 20:29 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.INFO.20241223-202309.34213.gz 4096 -rw-r--r--. 1 yugabyte yugabyte 148335 Dec 23 20:35 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.INFO.20241223-203101.45710 4 -rw-r--r--. 1 yugabyte yugabyte 825 Dec 23 20:24 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.WARNING.20241223-202310.34213.gz 4096 -rw-r--r--. 1 yugabyte yugabyte 5720 Dec 23 20:35 yb-tserver.yb-admin-sahith-uni-3-n1.yugabyte.log.WARNING.20241223-203101.45710 4 -rw-r--r--. 1 yugabyte yugabyte 262 Dec 23 20:23 ysql-conn-mgr-2024-12-23_202310.log.34420.gz 4 -rw-r--r--. 1 yugabyte yugabyte 204 Dec 23 20:30 ysql-conn-mgr-2024-12-23_203042.log.34420.gz 4 -rw-r--r--. 1 yugabyte yugabyte 264 Dec 23 20:31 ysql-conn-mgr-2024-12-23_203102.log.45786.gz 4096 -rw-r--r--. 1 yugabyte yugabyte 304 Dec 23 20:35 ysql-conn-mgr-2024-12-23_203434.log.45786 ``` As can be seen, the `ysql-conn-mgr` logs are zipped correctly. Test Plan: Manually tested that the zipped and unzipped logs are collected in support bundle. Adjusted UTs accordingly. Run UTs. Run itests. Reviewers: amalyshev, asharma Reviewed By: amalyshev, asharma Subscribers: yugaware Differential Revision: https://phorge.dev.yugabyte.com/D40884 --- managed/RUNTIME-FLAGS.md | 1 + .../templates/zip_purge_yb_logs.sh.j2 | 8 ++++ managed/devops/yb-server-ctl.yml | 13 +++++++ .../yw/common/config/UniverseConfKeys.java | 8 ++++ .../supportbundle/UniverseLogsComponent.java | 6 ++- managed/src/main/resources/reference.conf | 1 + .../yw/common/SupportBundleUtilTest.java | 39 ++++++++++++++++--- .../UniverseLogsComponentTest.java | 5 +++ 8 files changed, 75 insertions(+), 6 deletions(-) diff --git a/managed/RUNTIME-FLAGS.md b/managed/RUNTIME-FLAGS.md index d7bb0e43c78e..79821cd90b99 100644 --- a/managed/RUNTIME-FLAGS.md +++ b/managed/RUNTIME-FLAGS.md @@ -234,6 +234,7 @@ | "Minimum Incremental backup schedule frequency" | "yb.backup.minIncrementalScheduleFrequencyInSecs" | "UNIVERSE" | "Minimum Incremental backup schedule frequency in seconds" | "Long" | | "Universe logs regex pattern" | "yb.support_bundle.universe_logs_regex_pattern" | "UNIVERSE" | "Universe logs regex pattern in support bundle" | "String" | | "Postgres logs regex pattern" | "yb.support_bundle.postgres_logs_regex_pattern" | "UNIVERSE" | "Postgres logs regex pattern in support bundle" | "String" | +| "Connection Pooling logs regex pattern" | "yb.support_bundle.connection_pooling_logs_regex_pattern" | "UNIVERSE" | "Connection Pooling logs regex pattern in support bundle" | "String" | | "YSQL Upgrade Timeout in seconds" | "yb.upgrade.ysql_upgrade_timeout_sec" | "UNIVERSE" | "Controls the yb-client admin operation timeout when performing the runUpgradeYSQL subtask rpc calls." | "Integer" | | "Under replicated tablets check timeout" | "yb.checks.under_replicated_tablets.timeout" | "UNIVERSE" | "Controls the max time out when performing the checkUnderReplicatedTablets subtask" | "Duration" | | "Enabling under replicated tablets check" | "yb.checks.under_replicated_tablets.enabled" | "UNIVERSE" | "Controls whether or not to perform the checkUnderReplicatedTablets subtask" | "Boolean" | diff --git a/managed/devops/roles/configure-cluster-server/templates/zip_purge_yb_logs.sh.j2 b/managed/devops/roles/configure-cluster-server/templates/zip_purge_yb_logs.sh.j2 index c699a58c6254..166b68838674 100755 --- a/managed/devops/roles/configure-cluster-server/templates/zip_purge_yb_logs.sh.j2 +++ b/managed/devops/roles/configure-cluster-server/templates/zip_purge_yb_logs.sh.j2 @@ -34,6 +34,7 @@ ycql_audit_log_level=NONE logs_disk_percent_max=10 postgres_max_log_size_kb=$((100 * 1000)) +ysql_conn_mgr_max_log_size_kb=$((100 * 1000)) logs_purge_threshold_kb=$((10 * 1000000)) while [[ $# -gt 0 ]]; do @@ -201,12 +202,19 @@ for daemon_type in $daemon_types; do fi process_plain_files "$YB_LOG_DIR" "$plain_postgres_log" $plain_pg_logs_disk_usage_kb + plain_ysql_conn_mgr_log="ysql-conn-mgr*log* ! -name *.gz" + plain_ysql_conn_mgr_logs_disk_usage_kb=0 + process_plain_files "$YB_LOG_DIR" "$plain_ysql_conn_mgr_log" $plain_ysql_conn_mgr_logs_disk_usage_kb + if [ "$gzip_only" == false ]; then server_log="yb-$daemon_type*log.*" postgres_log="postgres*log*" + ysql_conn_mgr_log="ysql-conn-mgr*log*" delete_or_gzip_files "$YB_LOG_DIR" "$server_log" "$server_log.gz" \ "$permitted_disk_usage_kb" "delete" delete_or_gzip_files "$YB_LOG_DIR" "$postgres_log" "$postgres_log.gz" \ "$postgres_max_log_size_kb" "delete" + delete_or_gzip_files "$YB_LOG_DIR" "$ysql_conn_mgr_log" "$ysql_conn_mgr_log.gz" \ + "$ysql_conn_mgr_max_log_size_kb" "delete" fi done diff --git a/managed/devops/yb-server-ctl.yml b/managed/devops/yb-server-ctl.yml index e022fc5afde0..8622c59d53c6 100644 --- a/managed/devops/yb-server-ctl.yml +++ b/managed/devops/yb-server-ctl.yml @@ -69,6 +69,19 @@ - name: Reload daemon shell: "{{ systemd_prefix }} daemon-reload" when: command != "stop-destroy" + - name: Check if the logs cleanup script exists + stat: + path: "{{ yb_home_dir }}/bin/zip_purge_yb_logs.sh" + register: logs_cleanup_script + - name: Update the logs purge script if it already exists + template: + src: "roles/configure-cluster-server/templates/zip_purge_yb_logs.sh.j2" + dest: "{{ yb_home_dir }}/bin/zip_purge_yb_logs.sh" + force: yes + mode: 0755 + owner: "{{ user_name }}" + group: "{{ user_name }}" + when: logs_cleanup_script.stat.exists - name: Stop the {{ systemd_process }} process with systemd on destroy block: diff --git a/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java b/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java index a6e75d2d4a34..b6d3b48570b2 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java +++ b/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java @@ -709,6 +709,14 @@ public class UniverseConfKeys extends RuntimeConfigKeysModule { "Postgres logs regex pattern in support bundle", ConfDataType.StringType, ImmutableList.of(ConfKeyTags.PUBLIC)); + public static final ConfKeyInfo connectionPoolingLogsRegexPattern = + new ConfKeyInfo<>( + "yb.support_bundle.connection_pooling_logs_regex_pattern", + ScopeType.UNIVERSE, + "Connection Pooling logs regex pattern", + "Connection Pooling logs regex pattern in support bundle", + ConfDataType.StringType, + ImmutableList.of(ConfKeyTags.PUBLIC)); public static final ConfKeyInfo ysqlUpgradeTimeoutSec = new ConfKeyInfo<>( "yb.upgrade.ysql_upgrade_timeout_sec", diff --git a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java index 1721c349c0a5..5a6e761618c3 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java +++ b/managed/src/main/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponent.java @@ -103,7 +103,11 @@ public void downloadComponentBetweenDates( confGetter.getConfForScope(universe, UniverseConfKeys.universeLogsRegexPattern); String postgresLogsRegexPattern = confGetter.getConfForScope(universe, UniverseConfKeys.postgresLogsRegexPattern); - List fileRegexList = Arrays.asList(universeLogsRegexPattern, postgresLogsRegexPattern); + String connectionPoolingLogsRegexPattern = + confGetter.getConfForScope(universe, UniverseConfKeys.connectionPoolingLogsRegexPattern); + List fileRegexList = + Arrays.asList( + universeLogsRegexPattern, postgresLogsRegexPattern, connectionPoolingLogsRegexPattern); // Get and filter master log files that fall within given dates String masterLogsPath = nodeHomeDir + "/master/logs"; diff --git a/managed/src/main/resources/reference.conf b/managed/src/main/resources/reference.conf index ca239bf530f4..1f43bbc6316e 100644 --- a/managed/src/main/resources/reference.conf +++ b/managed/src/main/resources/reference.conf @@ -1099,6 +1099,7 @@ yb { default_mount_point_prefix = "/mnt/d" universe_logs_regex_pattern = "((?:.*)(?:yb-)(?:master|tserver)(?:.*))(\\d{8})-(?:\\d*)\\.(?:.*)" postgres_logs_regex_pattern = "((?:.*)(?:postgresql)-)(.{10})(?:.*)" + connection_pooling_logs_regex_pattern = "((?:.*)(?:ysql-conn-mgr)-)(.{10})(?:.*)" ybc_logs_regex_pattern = "((?:.*)(?:log)(?:.*))(\\d{8})-(?:\\d*)\\.(?:.*)" retention_days = 10 k8s_enabled = true diff --git a/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java b/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java index 30f54da0f36b..5f62a30305c6 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/SupportBundleUtilTest.java @@ -35,6 +35,8 @@ public class SupportBundleUtilTest extends FakeDBApplication { public static final String universe_logs_regex_pattern = "((?:.*)(?:yb-)(?:master|tserver)(?:.*))(\\d{8})-(?:\\d*)\\.(?:.*)"; public static final String postgres_logs_regex_pattern = "((?:.*)(?:postgresql)-)(.{10})(?:.*)"; + public static final String connection_pooling_logs_regex_pattern = + "((?:.*)(?:ysql-conn-mgr)-)(.{10})(?:.*)"; @Before public void setup() { @@ -129,18 +131,23 @@ public void testExtractFileTypeFromFileNameAndRegex() throws ParseException { "/mnt/disk0/yb-data/tserver/logs/" + "yb-tserver.yb-dev-sahith-new-yb-tserver-1.root.log.INFO.20221116-093807.24.gz", "/mnt/disk0/yb-data/master/logs/" - + "yb-master.yb-dev-sahith-new-yb-master-1.root.log.WARNING.20221116-093807.24.gz"); + + "yb-master.yb-dev-sahith-new-yb-master-1.root.log.WARNING.20221116-093807.24.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2024-12-24_065828.log.117963.gz"); List expectedFileTypeList = Arrays.asList( "/mnt/disk0/yb-data/tserver/logs/postgresql-", "/mnt/disk0/yb-data/tserver/logs/" + "yb-tserver.yb-dev-sahith-new-yb-tserver-1.root.log.INFO.", "/mnt/disk0/yb-data/master/logs/" - + "yb-master.yb-dev-sahith-new-yb-master-1.root.log.WARNING."); + + "yb-master.yb-dev-sahith-new-yb-master-1.root.log.WARNING.", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-"); for (int i = 0; i < fileNameList.size(); ++i) { List fileRegexList = - Arrays.asList(universe_logs_regex_pattern, postgres_logs_regex_pattern); + Arrays.asList( + universe_logs_regex_pattern, + postgres_logs_regex_pattern, + connection_pooling_logs_regex_pattern); assertEquals( expectedFileTypeList.get(i), supportBundleUtil.extractFileTypeFromFileNameAndRegex( @@ -152,6 +159,8 @@ public void testExtractFileTypeFromFileNameAndRegex() throws ParseException { public void testExtractDateFromFileNameAndRegex() throws ParseException { List fileNameList = Arrays.asList( + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2024-12-24_065828.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-01-01_000000.log.117900", "/mnt/disk0/yb-data/tserver/logs/postgresql-2022-11-16_093918.log.gz", "/mnt/disk0/yb-data/tserver/logs/postgresql-2021-12-31_000000.log.gz", "/mnt/disk0/yb-data/tserver/logs/postgresql-2020-01-01_000000.log.gz", @@ -170,6 +179,8 @@ public void testExtractDateFromFileNameAndRegex() throws ParseException { List expectedDateList = Arrays.asList( + "2024-12-24", + "2022-01-01", "2022-11-16", "2021-12-31", "2020-01-01", @@ -182,7 +193,10 @@ public void testExtractDateFromFileNameAndRegex() throws ParseException { for (int i = 0; i < fileNameList.size(); ++i) { List fileRegexList = - Arrays.asList(universe_logs_regex_pattern, postgres_logs_regex_pattern); + Arrays.asList( + universe_logs_regex_pattern, + postgres_logs_regex_pattern, + connection_pooling_logs_regex_pattern); Date date = new SimpleDateFormat("yyyy-MM-dd").parse(expectedDateList.get(i)); assertEquals( date, @@ -206,6 +220,13 @@ public void testFilterFilePathsBetweenDates() throws ParseException { "/mnt/disk0/yb-data/tserver/logs/postgresql-2022-11-20_000000.log.gz", "/mnt/disk0/yb-data/tserver/logs/postgresql-2022-11-21_000000.log.gz", "/mnt/disk0/yb-data/tserver/logs/postgresql-2022-11-22_000000.log", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2024-12-24_065828.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-18_065828.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-17_055828.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2020-01-02_065828.log.117963", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-22_065828.log.117963", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-20_000000.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-20_000001.log.117963", "/mnt/disk0/yb-data/tserver/logs/yb-tserver.yb-dev-sahith-new-yb-tserver-1.root." + "log.INFO.20221114-000000.00.gz", "/mnt/disk0/yb-data/tserver/logs/yb-tserver.yb-dev-sahith-new-yb-tserver-1.root." @@ -243,6 +264,11 @@ public void testFilterFilePathsBetweenDates() throws ParseException { "/mnt/disk0/yb-data/tserver/logs/postgresql-2022-11-20_000000.log.gz", "/mnt/disk0/yb-data/tserver/logs/postgresql-2022-11-21_000000.log.gz", "/mnt/disk0/yb-data/tserver/logs/postgresql-2022-11-22_000000.log", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-18_065828.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-17_055828.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-22_065828.log.117963", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-20_000000.log.117963.gz", + "/mnt/disk0/yb-data/tserver/logs/ysql-conn-mgr-2022-11-20_000001.log.117963", "/mnt/disk0/yb-data/tserver/logs/yb-tserver.yb-dev-sahith-new-yb-tserver-1.root." + "log.INFO.20221116-000000.00.gz", "/mnt/disk0/yb-data/tserver/logs/yb-tserver.yb-dev-sahith-new-yb-tserver-1.root." @@ -271,7 +297,10 @@ public void testFilterFilePathsBetweenDates() throws ParseException { List filteredLogFilePaths = supportBundleUtil.filterFilePathsBetweenDates( unFilteredLogFilePathList, - Arrays.asList(universe_logs_regex_pattern, postgres_logs_regex_pattern), + Arrays.asList( + universe_logs_regex_pattern, + postgres_logs_regex_pattern, + connection_pooling_logs_regex_pattern), startDate, endDate); diff --git a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java index 523277a1b2ba..d2444817e9b8 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/supportbundle/UniverseLogsComponentTest.java @@ -54,6 +54,8 @@ public class UniverseLogsComponentTest extends FakeDBApplication { private final String testUniverseLogsRegexPattern = "((?:.*)(?:yb-)(?:master|tserver)(?:.*))(\\d{8})-(?:\\d*)\\.(?:.*)"; private final String testPostgresLogsRegexPattern = "((?:.*)(?:postgresql)-)(.{10})(?:.*)"; + private final String testConnectionPoolingLogsRegexPattern = + "((?:.*)(?:ysql-conn-mgr)-)(.{10})(?:.*)"; private Universe universe; private Customer customer; private String fakeSupportBundleBasePath = "/tmp/yugaware_tests/support_bundle-universe_logs/"; @@ -102,6 +104,9 @@ public void setUp() throws Exception { when(MockConfGetter.getConfForScope( any(Universe.class), eq(UniverseConfKeys.postgresLogsRegexPattern))) .thenReturn(testPostgresLogsRegexPattern); + when(MockConfGetter.getConfForScope( + any(Universe.class), eq(UniverseConfKeys.connectionPoolingLogsRegexPattern))) + .thenReturn(testConnectionPoolingLogsRegexPattern); when(mockSupportBundleUtil.extractFileTypeFromFileNameAndRegex(any(), any())) .thenCallRealMethod(); when(mockSupportBundleUtil.extractDateFromFileNameAndRegex(any(), any())).thenCallRealMethod();