From 6846db43fab8eded97d37019026f272e7615491a Mon Sep 17 00:00:00 2001 From: lostsnow Date: Fri, 26 May 2023 16:07:06 +0800 Subject: [PATCH] reduce frequent log output --- .../main/java/io/dongtai/log/DongTaiLog.java | 100 ++++++++++++++---- .../main/java/io/dongtai/log/ErrorCode.java | 1 + .../java/io.dongtai.log/DongTaiLogTest.java | 52 +++++---- 3 files changed, 111 insertions(+), 42 deletions(-) diff --git a/dongtai-log/src/main/java/io/dongtai/log/DongTaiLog.java b/dongtai-log/src/main/java/io/dongtai/log/DongTaiLog.java index 128629854..aeaf43df6 100644 --- a/dongtai-log/src/main/java/io/dongtai/log/DongTaiLog.java +++ b/dongtai-log/src/main/java/io/dongtai/log/DongTaiLog.java @@ -2,8 +2,8 @@ import java.io.*; import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.regex.Matcher; @@ -23,9 +23,25 @@ public class DongTaiLog { private static final int YELLOW = 33; private static final int BLUE = 34; + // 5min + public static int FREQUENT_INTERVAL = 300000; + private static final String TITLE = "[io.dongtai.iast.agent] "; private static final String TITLE_COLOR = "[" + colorStr("io.dongtai.iast.agent", BLUE) + "] "; + private static final Set RESTRICTED_ERRORS = new HashSet(Arrays.asList( + ErrorCode.AGENT_MONITOR_COLLECT_PERFORMANCE_METRICS_FAILED, + ErrorCode.AGENT_MONITOR_CHECK_PERFORMANCE_METRICS_FAILED, + ErrorCode.AGENT_MONITOR_GET_DISK_USAGE_FAILED, + ErrorCode.REPORT_SEND_FAILED, + ErrorCode.REPLAY_REQUEST_FAILED, + ErrorCode.GRAPH_BUILD_AND_REPORT_FAILED, + ErrorCode.TAINT_COMMAND_GET_PARAMETERS_FAILED, + ErrorCode.TAINT_COMMAND_RANGE_PROCESS_FAILED + )); + + private static final ConcurrentHashMap ERROR_RECORD_MAP = new ConcurrentHashMap(); + static { if (System.console() != null && !System.getProperty("os.name").toLowerCase().contains("windows")) { ENABLE_COLOR = true; @@ -35,6 +51,35 @@ public class DongTaiLog { LOG_DIR = IastProperties.getLogDir(); } + private static class ErrorRecord { + private long lastWriteTime; + private int count; + + public ErrorRecord() { + this.lastWriteTime = new Date().getTime(); + this.count = 0; + } + + public boolean needWrite() { + long now = new Date().getTime(); + // 5min + return now - this.lastWriteTime > FREQUENT_INTERVAL; + } + + public int getCount() { + return this.count; + } + + public void incrementCount() { + this.count++; + } + + public void rotate() { + this.lastWriteTime = new Date().getTime(); + this.count = 0; + } + } + public static void configure(Integer agentId) throws Exception { ENABLED = IastProperties.isEnabled(); if (!ENABLED) { @@ -142,7 +187,7 @@ private static String colorStr(String msg, int colorCode) { return "\033[" + colorCode + "m" + msg + RESET; } - private static String getPrefix(LogLevel lvl, int code, boolean useColor) { + private static String getPrefix(LogLevel lvl, int code, int cnt, boolean useColor) { String prefix; if (useColor) { prefix = getTime() + TITLE_COLOR + lvl.getColorPrefix(); @@ -154,6 +199,10 @@ private static String getPrefix(LogLevel lvl, int code, boolean useColor) { prefix += "[" + String.valueOf(code) + "] "; } + if (cnt > 0) { + prefix += "[occurred " + String.valueOf(cnt) + " times] "; + } + return prefix; } @@ -168,11 +217,28 @@ private static String getMessage(String msg, Throwable t) { return msg; } - private static void log(LogLevel lvl, int code, String fmt, Object... arguments) { + private static void log(LogLevel lvl, ErrorCode ec, String fmt, Object... arguments) { if (!canLog(lvl)) { return; } + int cnt = 0; + if (RESTRICTED_ERRORS.contains(ec)) { + ErrorRecord er = ERROR_RECORD_MAP.get(ec); + if (er == null) { + ERROR_RECORD_MAP.put(ec, new ErrorRecord()); + } else { + if (!er.needWrite()) { + er.incrementCount(); + return; + } + + cnt = er.getCount(); + er.rotate(); + } + } + + int code = ec.getCode(); Throwable t = null; String msg = fmt; if (arguments.length == 1 && arguments[0] instanceof Throwable) { @@ -192,44 +258,36 @@ private static void log(LogLevel lvl, int code, String fmt, Object... arguments) if (msg.isEmpty()) { return; } - System.out.println(getPrefix(lvl, code, ENABLE_COLOR) + msg); - writeLogToFile(getPrefix(lvl, code, false) + msg, t); + System.out.println(getPrefix(lvl, code, cnt, ENABLE_COLOR) + msg); + writeLogToFile(getPrefix(lvl, code, cnt, false) + msg, t); } public static void trace(String fmt, Object... arguments) { - log(LogLevel.TRACE, 0, fmt, arguments); + log(LogLevel.TRACE, ErrorCode.NO_CODE, fmt, arguments); } public static void debug(String fmt, Object... arguments) { - log(LogLevel.DEBUG, 0, fmt, arguments); + log(LogLevel.DEBUG, ErrorCode.NO_CODE, fmt, arguments); } public static void info(String fmt, Object... arguments) { - log(LogLevel.INFO, 0, fmt, arguments); - } - - public static void warn(int code, String fmt, Object... arguments) { - log(LogLevel.WARN, code, fmt, arguments); + log(LogLevel.INFO, ErrorCode.NO_CODE, fmt, arguments); } public static void warn(ErrorCode ec, Object... arguments) { - log(LogLevel.WARN, ec.getCode(), ec.getMessage(), arguments); + log(LogLevel.WARN, ec, ec.getMessage(), arguments); } public static void warn(String format, Object... arguments) { - log(LogLevel.WARN, 0, format, arguments); - } - - public static void error(int code, String fmt, Object... arguments) { - log(LogLevel.ERROR, code, fmt, arguments); + log(LogLevel.WARN, ErrorCode.NO_CODE, format, arguments); } public static void error(ErrorCode ec, Object... arguments) { - log(LogLevel.ERROR, ec.getCode(), ec.getMessage(), arguments); + log(LogLevel.ERROR, ec, ec.getMessage(), arguments); } public static void error(String format, Object... arguments) { - log(LogLevel.ERROR, 0, format, arguments); + log(LogLevel.ERROR, ErrorCode.NO_CODE, format, arguments); } private static String format(String from, Object... arguments) { diff --git a/dongtai-log/src/main/java/io/dongtai/log/ErrorCode.java b/dongtai-log/src/main/java/io/dongtai/log/ErrorCode.java index 91ab9a415..91e497b9c 100644 --- a/dongtai-log/src/main/java/io/dongtai/log/ErrorCode.java +++ b/dongtai-log/src/main/java/io/dongtai/log/ErrorCode.java @@ -104,6 +104,7 @@ public enum ErrorCode { UTIL_TAINT_PARSE_CUSTOM_MODEL_FAILED(20612, "parse custom model {} getter {} failed"), UNKNOWN(99999, "unknown error"), + NO_CODE(0, "no error code"), ; private final int code; diff --git a/dongtai-log/src/test/java/io.dongtai.log/DongTaiLogTest.java b/dongtai-log/src/test/java/io.dongtai.log/DongTaiLogTest.java index 83620ff2c..eadbb9768 100644 --- a/dongtai-log/src/test/java/io.dongtai.log/DongTaiLogTest.java +++ b/dongtai-log/src/test/java/io.dongtai.log/DongTaiLogTest.java @@ -193,26 +193,6 @@ public void logTest() { Assert.assertEquals("ERROR log message with exception", TITLE + "[ERROR] foo, Exception: java.lang.Exception: bar" + LS, outputStreamCaptor.toString().substring(20)); - clear(); - DongTaiLog.error(110, "foo {} {}", "bar", "baz"); - Assert.assertEquals("ERROR log format", TITLE + "[ERROR] [110] foo bar baz" + LS, - outputStreamCaptor.toString().substring(20)); - clear(); - DongTaiLog.error(110, "foo {} {}", "bar", "baz", new Exception("bar")); - Assert.assertEquals("ERROR log format with code and exception", - TITLE + "[ERROR] [110] foo bar baz, Exception: java.lang.Exception: bar" + LS, - outputStreamCaptor.toString().substring(20)); - - clear(); - DongTaiLog.error(110, "foo {}", "bar", "baz", new Exception("bar")); - Assert.assertEquals("ERROR log format less with code and exception", - TITLE + "[ERROR] [110] foo bar, Exception: java.lang.Exception: bar" + LS, - outputStreamCaptor.toString().substring(20)); - clear(); - DongTaiLog.error(110, "foo {} {} {}", "bar", "baz", new Exception("bar")); - Assert.assertEquals("ERROR log format more with code and exception", - TITLE + "[ERROR] [110] foo bar baz {}, Exception: java.lang.Exception: bar" + LS, - outputStreamCaptor.toString().substring(20)); int code; String fmt; @@ -243,7 +223,6 @@ public void logTest() { clear(); DongTaiLog.error(ErrorCode.get("NOT EXISTS")); code = ErrorCode.UNKNOWN.getCode(); - fmt = String.format(ErrorCode.UNKNOWN.getMessage()); Assert.assertEquals("ERROR log with ErrorCode invalid name", TITLE + "[ERROR] [" + code + "] NOT EXISTS" + LS, outputStreamCaptor.toString().substring(20)); @@ -253,6 +232,37 @@ public void logTest() { TITLE + "[ERROR] [" + code + "] NOT EXISTS" + LS, outputStreamCaptor.toString().substring(20)); + // System.setOut(standardOut); + int fi = DongTaiLog.FREQUENT_INTERVAL; + DongTaiLog.FREQUENT_INTERVAL = 3000; + code = ErrorCode.REPORT_SEND_FAILED.getCode(); + fmt = String.format(ErrorCode.REPORT_SEND_FAILED.getMessage().replaceAll("\\{\\}", "%s"), "a", "b"); + for (int i = 0; i < 8; i++) { + clear(); + DongTaiLog.error(ErrorCode.REPORT_SEND_FAILED, "a", "b"); + if (i == 0) { + String msg = outputStreamCaptor.toString(); + Assert.assertTrue("ERROR log with frequent log " + i, msg.length() > 20); + Assert.assertEquals("ERROR log with frequent log " + i, + TITLE + "[ERROR] [" + code + "] " + fmt + LS, + msg.substring(20)); + } else if (i % 3 == 0) { + String msg = outputStreamCaptor.toString(); + Assert.assertTrue("ERROR log with frequent log " + i, msg.length() > 20); + Assert.assertEquals("ERROR log with frequent log " + i, + TITLE + "[ERROR] [" + code + "] [occurred 2 times] " + fmt + LS, + outputStreamCaptor.toString().substring(20)); + } else { + Assert.assertEquals("ERROR log with frequent log " + i, + "", outputStreamCaptor.toString()); + } + try { + Thread.sleep(1000); + } catch (InterruptedException ignore) { + } + } + DongTaiLog.FREQUENT_INTERVAL = fi; + clear(); } }