Change logrotate config

src/log-manager/deploy/log-manager.yaml.template

@@ -37,8 +37,8 @@ spec:
         image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}log-manager-logrotate:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
         imagePullPolicy: Always
         env:
-        - name: LOGROTATE_INTERVAL
-          value: "hourly"
+        - name: LOGROTATE_CRONSCHEDULE
+          value: "*/10 * * * *"
         volumeMounts:
         - name: pai-log
           mountPath: /usr/local/pai/logs

src/log-manager/src/logrotate/docker-entrypoint.sh

@@ -81,8 +81,13 @@ fi
 
 logrotate_cron_timetable="/usr/sbin/logrotate ${logrotate_parameters} --state=${logrotate_logstatus} /usr/bin/logrotate.d/logrotate.conf ${logrotate_cronlog}"
 
-# ----- Cron Start ------
+log_exist_time=30 # 30 day
+if [ -n "${LOG_EXIST_TIME}" ]; then
+  log_exist_time=${LOG_EXIST_TIME}
+fi
 
+# ----- Cron Start ------
+exec /usr/bin/go-cron '@daily' /bin/bash -c "find /usr/local/pai/logs/*/*/*/*/* -mtime +${log_exist_time} -type f -exec rm -fv {} \;"&
 if [ "$1" = 'cron' ]; then
   exec /usr/bin/go-cron "${logrotate_croninterval}" /bin/bash -c "${logrotate_cron_timetable}"
 fi

src/log-manager/src/logrotate/logrotate.conf

@@ -4,12 +4,11 @@
 /usr/local/pai/logs/*/*/*/*/*.all
 {
     nomail
-    weekly
-    rotate 4
-    compress
+    rotate 1
+    nocompress
     missingok
     notifempty
     copytruncate
-    maxsize 300M
+    size 256M
     maxage 30
 }

src/webportal/src/app/job/job-view/fabric/job-detail/conn.js

@@ -176,7 +176,7 @@ export async function getContainerLog(logUrl) {
     text: null,
   };
   const res = await fetch(logUrl);
-  const text = await res.text();
+  var text = await res.text();
   if (!res.ok) {
     throw new Error(res.statusText);
   }
@@ -222,6 +222,22 @@ export async function getContainerLog(logUrl) {
       throw new Error(`Log not available`);
     }
   } else if (config.logType === 'log-manager') {
+    // Try to get roated log if currently log content is less than 15KB
+    if (text.length <= 15 * 1024) {
+      const fullLogUrl = logUrl.replace('/tail/', '/full/');
+      const rotatedLogUrl = logUrl + '.1';
+      const rotatedLogRes = await fetch(rotatedLogUrl);
+      const fullLogRes = await fetch(fullLogUrl);
+      const rotatedText = await rotatedLogRes.text();
+      const fullLog = await fullLogRes.text();
+      if (rotatedLogRes.ok && rotatedText.trim() !== 'No such file!') {
+        text = rotatedText
+          .concat('\n--------log is rotated, may be lost during this--------\n')
+          .concat(fullLog);
+      }
+      // get last 16KB
+      text = text.slice(-16 * 1024);
+    }
     ret.text = text;
     ret.fullLogLink = logUrl.replace('/tail/', '/full/');
     return ret;