Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

reduce frequent log output #527

Merged
merged 1 commit into from
May 29, 2023
Merged

reduce frequent log output #527

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 79 additions & 21 deletions dongtai-log/src/main/java/io/dongtai/log/DongTaiLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.regex.Matcher;

Expand All @@ -23,9 +23,25 @@ public class DongTaiLog {
private static final int YELLOW = 33;
private static final int BLUE = 34;

// 5min
public static int FREQUENT_INTERVAL = 300000;

private static final String TITLE = "[io.dongtai.iast.agent] ";
private static final String TITLE_COLOR = "[" + colorStr("io.dongtai.iast.agent", BLUE) + "] ";

private static final Set<ErrorCode> RESTRICTED_ERRORS = new HashSet<ErrorCode>(Arrays.asList(
ErrorCode.AGENT_MONITOR_COLLECT_PERFORMANCE_METRICS_FAILED,
ErrorCode.AGENT_MONITOR_CHECK_PERFORMANCE_METRICS_FAILED,
ErrorCode.AGENT_MONITOR_GET_DISK_USAGE_FAILED,
ErrorCode.REPORT_SEND_FAILED,
ErrorCode.REPLAY_REQUEST_FAILED,
ErrorCode.GRAPH_BUILD_AND_REPORT_FAILED,
ErrorCode.TAINT_COMMAND_GET_PARAMETERS_FAILED,
ErrorCode.TAINT_COMMAND_RANGE_PROCESS_FAILED
));

private static final ConcurrentHashMap<ErrorCode, ErrorRecord> ERROR_RECORD_MAP = new ConcurrentHashMap<ErrorCode, ErrorRecord>();

static {
if (System.console() != null && !System.getProperty("os.name").toLowerCase().contains("windows")) {
ENABLE_COLOR = true;
Expand All @@ -35,6 +51,35 @@ public class DongTaiLog {
LOG_DIR = IastProperties.getLogDir();
}

private static class ErrorRecord {
private long lastWriteTime;
private int count;

public ErrorRecord() {
this.lastWriteTime = new Date().getTime();
this.count = 0;
}

public boolean needWrite() {
long now = new Date().getTime();
// 5min
return now - this.lastWriteTime > FREQUENT_INTERVAL;
}

public int getCount() {
return this.count;
}

public void incrementCount() {
this.count++;
}

public void rotate() {
this.lastWriteTime = new Date().getTime();
this.count = 0;
}
}

public static void configure(Integer agentId) throws Exception {
ENABLED = IastProperties.isEnabled();
if (!ENABLED) {
Expand Down Expand Up @@ -142,7 +187,7 @@ private static String colorStr(String msg, int colorCode) {
return "\033[" + colorCode + "m" + msg + RESET;
}

private static String getPrefix(LogLevel lvl, int code, boolean useColor) {
private static String getPrefix(LogLevel lvl, int code, int cnt, boolean useColor) {
String prefix;
if (useColor) {
prefix = getTime() + TITLE_COLOR + lvl.getColorPrefix();
Expand All @@ -154,6 +199,10 @@ private static String getPrefix(LogLevel lvl, int code, boolean useColor) {
prefix += "[" + String.valueOf(code) + "] ";
}

if (cnt > 0) {
prefix += "[occurred " + String.valueOf(cnt) + " times] ";
}

return prefix;
}

Expand All @@ -168,11 +217,28 @@ private static String getMessage(String msg, Throwable t) {
return msg;
}

private static void log(LogLevel lvl, int code, String fmt, Object... arguments) {
private static void log(LogLevel lvl, ErrorCode ec, String fmt, Object... arguments) {
if (!canLog(lvl)) {
return;
}

int cnt = 0;
if (RESTRICTED_ERRORS.contains(ec)) {
ErrorRecord er = ERROR_RECORD_MAP.get(ec);
if (er == null) {
ERROR_RECORD_MAP.put(ec, new ErrorRecord());
} else {
if (!er.needWrite()) {
er.incrementCount();
return;
}

cnt = er.getCount();
er.rotate();
}
}

int code = ec.getCode();
Throwable t = null;
String msg = fmt;
if (arguments.length == 1 && arguments[0] instanceof Throwable) {
Expand All @@ -192,44 +258,36 @@ private static void log(LogLevel lvl, int code, String fmt, Object... arguments)
if (msg.isEmpty()) {
return;
}
System.out.println(getPrefix(lvl, code, ENABLE_COLOR) + msg);
writeLogToFile(getPrefix(lvl, code, false) + msg, t);
System.out.println(getPrefix(lvl, code, cnt, ENABLE_COLOR) + msg);
writeLogToFile(getPrefix(lvl, code, cnt, false) + msg, t);
}

public static void trace(String fmt, Object... arguments) {
log(LogLevel.TRACE, 0, fmt, arguments);
log(LogLevel.TRACE, ErrorCode.NO_CODE, fmt, arguments);
}

public static void debug(String fmt, Object... arguments) {
log(LogLevel.DEBUG, 0, fmt, arguments);
log(LogLevel.DEBUG, ErrorCode.NO_CODE, fmt, arguments);
}

public static void info(String fmt, Object... arguments) {
log(LogLevel.INFO, 0, fmt, arguments);
}

public static void warn(int code, String fmt, Object... arguments) {
log(LogLevel.WARN, code, fmt, arguments);
log(LogLevel.INFO, ErrorCode.NO_CODE, fmt, arguments);
}

public static void warn(ErrorCode ec, Object... arguments) {
log(LogLevel.WARN, ec.getCode(), ec.getMessage(), arguments);
log(LogLevel.WARN, ec, ec.getMessage(), arguments);
}

public static void warn(String format, Object... arguments) {
log(LogLevel.WARN, 0, format, arguments);
}

public static void error(int code, String fmt, Object... arguments) {
log(LogLevel.ERROR, code, fmt, arguments);
log(LogLevel.WARN, ErrorCode.NO_CODE, format, arguments);
}

public static void error(ErrorCode ec, Object... arguments) {
log(LogLevel.ERROR, ec.getCode(), ec.getMessage(), arguments);
log(LogLevel.ERROR, ec, ec.getMessage(), arguments);
}

public static void error(String format, Object... arguments) {
log(LogLevel.ERROR, 0, format, arguments);
log(LogLevel.ERROR, ErrorCode.NO_CODE, format, arguments);
}

private static String format(String from, Object... arguments) {
Expand Down
1 change: 1 addition & 0 deletions dongtai-log/src/main/java/io/dongtai/log/ErrorCode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ public enum ErrorCode {
UTIL_TAINT_PARSE_CUSTOM_MODEL_FAILED(20612, "parse custom model {} getter {} failed"),

UNKNOWN(99999, "unknown error"),
NO_CODE(0, "no error code"),
;

private final int code;
Expand Down
52 changes: 31 additions & 21 deletions dongtai-log/src/test/java/io.dongtai.log/DongTaiLogTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,26 +193,6 @@ public void logTest() {
Assert.assertEquals("ERROR log message with exception",
TITLE + "[ERROR] foo, Exception: java.lang.Exception: bar" + LS,
outputStreamCaptor.toString().substring(20));
clear();
DongTaiLog.error(110, "foo {} {}", "bar", "baz");
Assert.assertEquals("ERROR log format", TITLE + "[ERROR] [110] foo bar baz" + LS,
outputStreamCaptor.toString().substring(20));
clear();
DongTaiLog.error(110, "foo {} {}", "bar", "baz", new Exception("bar"));
Assert.assertEquals("ERROR log format with code and exception",
TITLE + "[ERROR] [110] foo bar baz, Exception: java.lang.Exception: bar" + LS,
outputStreamCaptor.toString().substring(20));

clear();
DongTaiLog.error(110, "foo {}", "bar", "baz", new Exception("bar"));
Assert.assertEquals("ERROR log format less with code and exception",
TITLE + "[ERROR] [110] foo bar, Exception: java.lang.Exception: bar" + LS,
outputStreamCaptor.toString().substring(20));
clear();
DongTaiLog.error(110, "foo {} {} {}", "bar", "baz", new Exception("bar"));
Assert.assertEquals("ERROR log format more with code and exception",
TITLE + "[ERROR] [110] foo bar baz {}, Exception: java.lang.Exception: bar" + LS,
outputStreamCaptor.toString().substring(20));

int code;
String fmt;
Expand Down Expand Up @@ -243,7 +223,6 @@ public void logTest() {
clear();
DongTaiLog.error(ErrorCode.get("NOT EXISTS"));
code = ErrorCode.UNKNOWN.getCode();
fmt = String.format(ErrorCode.UNKNOWN.getMessage());
Assert.assertEquals("ERROR log with ErrorCode invalid name",
TITLE + "[ERROR] [" + code + "] NOT EXISTS" + LS,
outputStreamCaptor.toString().substring(20));
Expand All @@ -253,6 +232,37 @@ public void logTest() {
TITLE + "[ERROR] [" + code + "] NOT EXISTS" + LS,
outputStreamCaptor.toString().substring(20));

// System.setOut(standardOut);
int fi = DongTaiLog.FREQUENT_INTERVAL;
DongTaiLog.FREQUENT_INTERVAL = 3000;
code = ErrorCode.REPORT_SEND_FAILED.getCode();
fmt = String.format(ErrorCode.REPORT_SEND_FAILED.getMessage().replaceAll("\\{\\}", "%s"), "a", "b");
for (int i = 0; i < 8; i++) {
clear();
DongTaiLog.error(ErrorCode.REPORT_SEND_FAILED, "a", "b");
if (i == 0) {
String msg = outputStreamCaptor.toString();
Assert.assertTrue("ERROR log with frequent log " + i, msg.length() > 20);
Assert.assertEquals("ERROR log with frequent log " + i,
TITLE + "[ERROR] [" + code + "] " + fmt + LS,
msg.substring(20));
} else if (i % 3 == 0) {
String msg = outputStreamCaptor.toString();
Assert.assertTrue("ERROR log with frequent log " + i, msg.length() > 20);
Assert.assertEquals("ERROR log with frequent log " + i,
TITLE + "[ERROR] [" + code + "] [occurred 2 times] " + fmt + LS,
outputStreamCaptor.toString().substring(20));
} else {
Assert.assertEquals("ERROR log with frequent log " + i,
"", outputStreamCaptor.toString());
}
try {
Thread.sleep(1000);
} catch (InterruptedException ignore) {
}
}
DongTaiLog.FREQUENT_INTERVAL = fi;

clear();
}
}