Merge pull request #306 from Nizernizer/main

fix bug

dongtai-agent/src/main/java/io/dongtai/iast/agent/Agent.java

@@ -107,9 +107,9 @@ private static void doAttach(String pid, String agentArgs) {
                 DongTaiLog.error("attach failure, please try again with command: {}", Arrays.toString(execution));
             }
         } catch (IOException e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.agent.Agent.doAttach(java.lang.String,java.lang.String)",e);
         } catch (InterruptedException e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.agent.Agent.doAttach(java.lang.String,java.lang.String)",e);
         }
     }
 

dongtai-agent/src/main/java/io/dongtai/iast/agent/IastProperties.java

@@ -47,7 +47,7 @@ private IastProperties() {
         try {
             init();
         } catch (ClassNotFoundException e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.agent.IastProperties.<init>()",e);
         }
     }
 

dongtai-agent/src/main/java/io/dongtai/iast/agent/util/base64/Base64Decoder.java

@@ -26,7 +26,7 @@ public static byte[] decodeBase64FromString(String data) {
         try {
             return instance.decodeBuffer(data);
         } catch (IOException e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.agent.util.base64.Base64Decoder.decodeBase64FromString(java.lang.String)",e);
         }
         return null;
     }

dongtai-agent/src/main/java/io/dongtai/iast/agent/util/http/HttpClientUtils.java

@@ -114,6 +114,7 @@ private static StringBuilder sendRequest(HttpMethods method, String baseUrl, Str
                 response.append('\r');
             }
             rd.close();
+            DongTaiLog.debug("dongtai upload url is {}, request is {} ,response is {}", urlStr, data, response.toString());
             return response;
         } catch (Exception e){
             DongTaiLog.error(e);

dongtai-agent/src/main/resources/bin/fluent.conf

@@ -11,6 +11,7 @@
   Tag          Dongtai-${HOSTNAME_AGENT_ID}.log
   Buffer_Chunk_Size 320KB
   Buffer_Max_Size   520KB
+  Read_from_Head    true
 [FILTER]
   Name         record_modifier
   Match        *

dongtai-api/src/main/java/io/dongtai/api/servlet2/ServletRequestWrapper.java

@@ -55,7 +55,7 @@ public ServletInputStream getInputStream() throws IOException {
                     bufferedReader.close();
                 } catch (IOException e) {
                     // fixme: add logger for solve exception
-                    DongTaiLog.error(e);
+                    DongTaiLog.error("io.dongtai.api.servlet2.ServletRequestWrapper.getInputStream()",e);
                 }
                 body = stringBuilder.toString();
                 isCachedBody = true;

dongtai-core/src/main/java/io/dongtai/iast/core/EngineManager.java

@@ -87,7 +87,7 @@ public static void openHookPointFallback(String className, String method, String
         final double limitRate = EngineManager.getFallbackManager().getHookRateLimiter().getRate();
         DongTaiLog.debug("HookPoint rate limit! hookType: " + hookType + ", method:" + className + "." + method
                 + ", sign:" + methodSign + " ,rate:" + limitRate);
-        HookPointRateLimitReport.sendReport(className, method, methodSign, hookType, limitRate);
+//        HookPointRateLimitReport.sendReport(className, method, methodSign, hookType, limitRate);
         FallbackSwitch.setHeavyHookFallback(true);
     }
 

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/fallback/FallbackSwitch.java

@@ -46,6 +46,13 @@ private FallbackSwitch() {
     @Setter
     private static boolean PERFORMANCE_FALLBACK = false;
 
+    /**
+     * URL 限流开关
+     */
+    @Getter
+    @Setter
+    public static boolean URL_FALLBACK = false;
+
     // *************************************************************
     // 二次降级配置
     // *************************************************************

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/fallback/breaker/HeavyTrafficBreaker.java

@@ -67,7 +67,7 @@ protected void initBreaker(Properties cfg) {
                     CircuitBreaker.State state = event.getStateTransition().getToState();
                     if (state == CircuitBreaker.State.OPEN) {
                         FallbackSwitch.setHeavyTrafficLimitFallback(true);
-                        HeavyTrafficRateLimitReport.sendReport(trafficLimitRate);
+//                        HeavyTrafficRateLimitReport.sendReport(trafficLimitRate);
                     }
                     // 因为本断路器的样本来自流量，打开后无法获取新样本，故需要在 HALF_OPEN 状态直接转到 CLOSE 状态
                     if (state == CircuitBreaker.State.HALF_OPEN) {

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/fallback/report/PerformanceLimitReport.java

@@ -23,8 +23,8 @@ public static void appendPerformanceBreakLog(PerformanceBreakReportBody.Performa
     public static void sendReport() {
         PERFORMANCE_BREAK_REPORT_BODY.getDetail().setAgentId(EngineManager.getAgentId());
         PERFORMANCE_BREAK_REPORT_BODY.getDetail().setBreakDate(new Date());
-        String report = GsonUtils.toJson(PERFORMANCE_BREAK_REPORT_BODY);
-        sendReport(report);
+/*        String report = GsonUtils.toJson(PERFORMANCE_BREAK_REPORT_BODY);
+        sendReport(report);*/
         PERFORMANCE_BREAK_REPORT_BODY.clearAllPerformanceBreakLog();
     }
 

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/fallback/report/SecondFallbackReport.java

@@ -27,7 +27,7 @@ public static boolean isSecondFallbackLogEmpty() {
      */
     public static void sendReport(){
         FALLBACK_REPORT_LOG.getDetail().setAgentId(EngineManager.getAgentId());
-        sendReport(GsonUtils.toJson(FALLBACK_REPORT_LOG));
+//        sendReport(GsonUtils.toJson(FALLBACK_REPORT_LOG));
         FALLBACK_REPORT_LOG.clear();
     }
 

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/spring/SpringApplicationImpl.java

@@ -41,7 +41,7 @@ private static void loadApplicationContext() {
                 proxyClass = iastClassLoader.loadClass("cn.huoxian.iast.spring.SpringApplicationContext");
                 getAPI = proxyClass.getDeclaredMethod("getAPI", Object.class);
             } catch (NoSuchMethodException e) {
-                DongTaiLog.error(e);
+                DongTaiLog.error("io.dongtai.iast.core.bytecode.enhance.plugin.spring.SpringApplicationImpl.loadApplicationContext()",e);
             }
         }
     }

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/SpyDispatcherImpl.java

@@ -23,6 +23,7 @@
 public class SpyDispatcherImpl implements SpyDispatcher {
 
     public static final AtomicInteger INVOKE_ID_SEQUENCER = new AtomicInteger(1);
+    private static final ThreadLocal<Long> responseTime = new ThreadLocal<>();
 
     /**
      * mark for enter Http Entry Point
@@ -55,6 +56,8 @@ public void leaveHttp(Object request, Object response) {
                     EngineManager.maintainRequestCount();
                     GraphBuilder.buildAndReport(request, response);
                     EngineManager.cleanThreadState();
+                    long responseTimeEnd = System.currentTimeMillis()-responseTime.get()+8;
+                    DongTaiLog.debug(GraphBuilder.getURL() + " response time: "+responseTimeEnd+"ms");
                 }
 
                 EngineManager.turnOnDongTai();
@@ -76,6 +79,7 @@ public void leaveHttp(Object request, Object response) {
     @Override
     public boolean isFirstLevelHttp() {
         try {
+            responseTime.set(System.currentTimeMillis());
             return EngineManager.isEngineRunning() && EngineManager.SCOPE_TRACKER
                     .isFirstLevelHttp();
         } catch (Exception e) {

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/controller/impl/HttpImpl.java

@@ -55,10 +55,8 @@ private static void createClassLoader(Object req) {
                 cloneResponseMethod = CLASS_OF_SERVLET_PROXY
                         .getDeclaredMethod("cloneResponse", Object.class, boolean.class);
             }
-        } catch (MalformedURLException e) {
-            DongTaiLog.error(e);
-        } catch (NoSuchMethodException e) {
-            DongTaiLog.error(e);
+        } catch (MalformedURLException | NoSuchMethodException e) {
+            DongTaiLog.error("io.dongtai.iast.core.handler.hookpoint.controller.impl.HttpImpl.createClassLoader(java.lang.Object)",e);
         }
     }
 

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/graphy/GraphBuilder.java

@@ -22,6 +22,8 @@
  */
 public class GraphBuilder {
 
+    private static String URL;
+
     public static void buildAndReport(Object request, Object response) {
         List<GraphNode> nodeList = build();
         String report = convertToReport(nodeList, request, response);
@@ -93,8 +95,11 @@ public static String convertToReport(List<GraphNode> nodeList, Object request, O
         detail.put(ReportConstant.SCHEME, requestMeta.getOrDefault("scheme", ""));
         detail.put(ReportConstant.METHOD, requestMeta.getOrDefault("method", ""));
         detail.put(ReportConstant.SECURE, requestMeta.getOrDefault("secure", ""));
-        detail.put(ReportConstant.URL, requestMeta.getOrDefault("requestURL", "").toString());
-        detail.put(ReportConstant.URI, requestMeta.getOrDefault("requestURI", ""));
+        String requestURL = requestMeta.getOrDefault("requestURL", "").toString();
+        detail.put(ReportConstant.URL, requestURL);
+        String requestURI = requestMeta.getOrDefault("requestURI", "").toString();
+        detail.put(ReportConstant.URI, requestURI);
+        setURL(requestURL);
         detail.put(ReportConstant.CLIENT_IP, requestMeta.getOrDefault("remoteAddr", ""));
         detail.put(ReportConstant.QUERY_STRING, requestMeta.getOrDefault("queryString", ""));
         detail.put(ReportConstant.REQ_HEADER,
@@ -131,4 +136,12 @@ private static byte[] getResponseBody(Map<String, Object> responseMeta) {
             return responseBody;
         }
     }
+
+    public static String getURL() {
+        return URL;
+    }
+
+    public static void setURL(String URL) {
+        GraphBuilder.URL = URL;
+    }
 }

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/vulscan/normal/CookieFlagsMissingVulScan.java

@@ -25,7 +25,7 @@ public void scan(IastSinkModel sink, MethodEvent event) {
                 sendReport(getLatestStack(), sink.getType());
                 break;
             } catch (Exception e) {
-                DongTaiLog.error(e);
+                DongTaiLog.error("io.dongtai.iast.core.handler.hookpoint.vulscan.normal.CookieFlagsMissingVulScan.scan(io.dongtai.iast.core.handler.hookpoint.models.IastSinkModel,io.dongtai.iast.core.handler.hookpoint.models.MethodEvent)",e);
             }
         }
     }

dongtai-core/src/main/java/io/dongtai/iast/core/replay/HttpRequestReplay.java

@@ -59,7 +59,7 @@ private static void doReplay(IastReplayModel replayModel) {
                 sendRequest(replayModel.getRequestMethod(), url, replayModel.getRequestBody(), headers);
             }
         } catch (Exception e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.core.replay.HttpRequestReplay.doReplay(io.dongtai.iast.core.handler.hookpoint.models.IastReplayModel)",e);
         }
     }
 
@@ -133,7 +133,7 @@ private static void sendRequest(String method, String fullUrl, String data, Hash
             rd.close();
             DongTaiLog.debug("Request replay response: {}",response);
         } catch (Exception e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.core.replay.HttpRequestReplay.sendRequest(java.lang.String,java.lang.String,java.lang.String,java.util.HashMap<java.lang.String,java.lang.String>)",e);
         } finally {
             if (connection != null) {
                 connection.disconnect();

dongtai-core/src/main/java/io/dongtai/iast/core/utils/ConfigUtils.java

@@ -46,7 +46,7 @@ public static Set<String>[] loadConfigFromFile(String filename) {
             }
         } catch (IOException e) {
             DongTaiLog.error("读取配置文件：{} 失败，错误信息：{}", filename, e);
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.core.utils.ConfigUtils.loadConfigFromFile(java.lang.String)",e);
         }
         return new HashSet[]{container, startWith, endWith};
     }

dongtai-core/src/main/java/io/dongtai/iast/core/utils/Constants.java

@@ -4,7 +4,7 @@
  * @author dongzhiyong@huoxian.cn
  */
 public class Constants {
-    public static final String AGENT_VERSION_VALUE = "v1.5.0";
+    public static final String AGENT_VERSION_VALUE = "v1.7.0.test.2";
     public final static String API_REPORT_UPLOAD = "/api/v1/report/upload";
     public final static String SERVER_ADDRESS = "/api/v1/agent/update";
     public final static String API_HOOK_PROFILE = "/api/v1/profiles";

dongtai-core/src/main/java/io/dongtai/iast/core/utils/HttpClientUtils.java

@@ -43,7 +43,6 @@ public static StringBuilder sendGet(String uri, String arg, String value) {
     public static StringBuilder sendPost(String uri, String value) {
         StringBuilder response;
         response = sendRequest(HttpMethods.POST, PROPERTIES.getBaseUrl(), uri, value, null, PROXY);
-        DongTaiLog.debug("dongtai upload url is {}, resp is {}", uri, response.toString());
         return response;
     }
 
@@ -101,6 +100,7 @@ private static StringBuilder sendRequest(HttpMethods method, String baseUrl, Str
                 response.append('\r');
             }
             rd.close();
+            DongTaiLog.debug("dongtai upload url is {}, request is {} ,response is {}", urlStr, data, response.toString());
             return response;
         } catch (Exception e) {
             DongTaiLog.error(e);

dongtai-core/src/main/java/io/dongtai/iast/core/utils/PropertyUtils.java

@@ -64,10 +64,8 @@ private void init() {
                 cfg.load(fis);
                 fis.close();
             }
-        } catch (FileNotFoundException e) {
-            DongTaiLog.error(e);
         } catch (IOException e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.core.utils.PropertyUtils.init()",e);
         }
     }
 

dongtai-core/src/main/java/io/dongtai/iast/core/utils/base64/Base64Decoder.java

@@ -26,7 +26,7 @@ public static byte[] decodeBase64FromString(String data) {
         try {
             return instance.decodeBuffer(data);
         } catch (IOException e) {
-            DongTaiLog.error(e);
+            DongTaiLog.error("io.dongtai.iast.core.utils.base64.Base64Decoder.decodeBase64FromString(java.lang.String)",e);
         }
         return null;
     }

dongtai-core/src/main/java/io/dongtai/iast/core/utils/matcher/ConfigMatcher.java

@@ -1,5 +1,6 @@
 package io.dongtai.iast.core.utils.matcher;
 
+import io.dongtai.iast.core.bytecode.enhance.plugin.fallback.FallbackSwitch;
 import io.dongtai.iast.core.utils.ConfigUtils;
 import io.dongtai.iast.core.utils.PropertyUtils;
 import io.dongtai.log.DongTaiLog;
@@ -29,7 +30,7 @@ public class ConfigMatcher {
     private final AbstractMatcher SERVER_CLASS = new ServerClass();
     private Instrumentation inst;
 
-    private final Set<String> BLACK_URL;
+    public final Set<String> BLACK_URL;
 
     public static ConfigMatcher getInstance() {
         if (null == INSTANCE) {
@@ -91,6 +92,10 @@ public boolean getBlackUrl(Map<String, Object> request) {
                         if (null != headers.get(strings[0].toLowerCase())) {
                             return true;
                         }
+                    case 3:
+                        if (FallbackSwitch.URL_FALLBACK && uri.contains(strings[0])){
+                            return true;
+                        }
                     default:
                         continue;
                 }

dongtai-core/src/main/resources/com.secnium.iast.resources/blackurl.txt

@@ -5,4 +5,6 @@ isTest 2
 /internalHeartBeat/_check 1
 /actuator/health 1
 /service/lessonmenu.mvc 1
-/service/lessonoverview.mvc 1
+/service/lessonoverview.mvc 1
+.map 1
+.lesson 1

dongtai-log/src/main/java/io/dongtai/log/DongTaiLog.java

@@ -247,6 +247,22 @@ public static void error(Throwable t) {
         }
     }
 
+    public static void error(String des, Throwable t) {
+        if (canLog(Level.FINER) && enablePrintLog) {
+            String msg = des+"\n"+t.getMessage();
+            if (enableColor) {
+                System.out.println(getTime() + TITTLE_COLOR_PREFIX + ERROR_COLOR_PREFIX + msg);
+            } else {
+                System.out.println(getTime() + TITTLE + ERROR_PREFIX + msg);
+            }
+            msg = getTime() + TITTLE + ERROR_PREFIX + msg;
+            StringWriter stringWriter = new StringWriter();
+            t.printStackTrace(new PrintWriter(stringWriter));
+            msg = msg + stringWriter;
+            writeLogToFile(msg);
+        }
+    }
+
     private static String format(String from, Object... arguments) {
         if (from != null) {
             String computed = from;