+

pkgforge · Oct 9, 2024 · 612fda0 · 612fda0
1 parent c94a52a
commit 612fda0
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 34 deletions.
diff --git a/.github/workflows/build_aarch64_Linux.yaml b/.github/workflows/build_aarch64_Linux.yaml
@@ -177,24 +177,26 @@ jobs:
           bash "${SYSTMP}/BUILD.sh" | tee "${SYSTMP}/BUILD.log" 2>&1
           ##Run without STDOUT/Logs
           #bash "${SYSTMP}/BUILD.sh" >/dev/null 2>&1
-          ##Purge Tokens (in case set -x & gh didn't redact)
-          cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.log.txt"
-          sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          ##Fetch Only Relevant Logs
-          if grep -iq "completed" "${SYSTMP}/BUILD.log.txt"; then
-             mv "${SYSTMP}/BUILD.log.txt" "${SYSTMP}/BUILD.log"
-          elif grep -iq "initializing" "${SYSTMP}/BUILD.log.txt"; then
-             mv "${SYSTMP}/BUILD.log.txt" "${SYSTMP}/BUILD_FAILED.log"
+          if [[ -f "${SYSTMP}/BUILD.log" ]] && [[ $(stat -c%s "${SYSTMP}/BUILD.log") -gt 1024 ]]; then
+             ##Purge Tokens (in case set -x & gh didn't redact)
+             cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.gh.log.txt"
+             sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             cat "${SYSTMP}/BUILD.gh.log.txt" > "${SYSTMP}/BUILD.log"
+             if grep -iq "completed" "${SYSTMP}/BUILD.gh.log.txt"; then
+                mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD.log"
+             elif grep -iq "initializing" "${SYSTMP}/BUILD.gh.log.txt"; then
+                mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD_FAILED.log"
+             fi
+             ##rClone Upload logs
+             rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
+             rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD_FAILED.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
           fi
-          ##rClone Upload logs
-          rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
-          rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/BUILD_FAILED.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
         continue-on-error: true
 #------------------------------------------------------------------------------------#
       #Logs & Artifacts  

diff --git a/.github/workflows/build_x86_64_Linux.yaml b/.github/workflows/build_x86_64_Linux.yaml
@@ -177,24 +177,26 @@ jobs:
           bash "${SYSTMP}/BUILD.sh" | tee "${SYSTMP}/BUILD.log" 2>&1
           ##Run without STDOUT/Logs
           #bash "${SYSTMP}/BUILD.sh" >/dev/null 2>&1
-          ##Purge Tokens (in case set -x & gh didn't redact)
-          cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.log.txt"
-          sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.log" 2>/dev/null
-          ##Fetch Only Relevant Logs
-          if grep -iq "completed" "${SYSTMP}/BUILD.log.txt"; then
-             mv "${SYSTMP}/BUILD.log.txt" "${SYSTMP}/BUILD.log"
-          elif grep -iq "initializing" "${SYSTMP}/BUILD.log.txt"; then
-             mv "${SYSTMP}/BUILD.log.txt" "${SYSTMP}/BUILD_FAILED.log"
+          if [[ -f "${SYSTMP}/BUILD.log" ]] && [[ $(stat -c%s "${SYSTMP}/BUILD.log") -gt 1024 ]]; then
+             ##Purge Tokens (in case set -x & gh didn't redact)
+             cat "${SYSTMP}/BUILD.log" | ansi2txt > "${SYSTMP}/BUILD.gh.log.txt"
+             sed -i '/.*github_pat.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*ghp_.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*access_key_id.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*secret_access_key.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*token.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             sed -i '/.*cloudflarestorage.*/Id' "${SYSTMP}/BUILD.gh.log.txt" 2>/dev/null
+             ##Fetch Only Relevant Logs
+             if grep -iq "completed" "${SYSTMP}/BUILD.gh.log.txt"; then
+                mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD.log"
+             elif grep -iq "initializing" "${SYSTMP}/BUILD.gh.log.txt"; then
+                mv "${SYSTMP}/BUILD.gh.log.txt" "${SYSTMP}/BUILD_FAILED.log"
+             fi
+             ##rClone Upload logs
+             rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
+             rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD_FAILED.gh.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
           fi
-          ##rClone Upload logs
-          rclone copyto "${SYSTMP}/BUILD.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
-          rclone copyto "${SYSTMP}/BUILD_FAILED.log" "r2:/pub/repos/Azathothas/Toolpacks-Extras/BUILD_FAILED.log.txt" --user-agent="${USER_AGENT}" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress
         continue-on-error: true
 #------------------------------------------------------------------------------------#
       #Logs & Artifacts  

diff --git a/.github/workflows/healthchecks_housekeeping.yaml b/.github/workflows/healthchecks_housekeeping.yaml
@@ -118,6 +118,8 @@ jobs:
           pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1
            mkdir -p "${GITHUB_WORKSPACE}/main/aarch64-Linux"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/BUILD.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.log.txt"
+           curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD.gh.log.txt"
+           curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/aarch64-Linux/BUILD_FAILED.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BUILD_FAILED.gh.log.txt"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/BLAKE3SUM.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/BLAKE3SUM.json"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/FILE.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/FILE.json"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/aarch64-Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/aarch64-Linux/METADATA.json"
@@ -135,6 +137,8 @@ jobs:
           pushd "${GITHUB_WORKSPACE}/main" >/dev/null 2>&1
            mkdir -p "${GITHUB_WORKSPACE}/main/x86_64-Linux"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/BUILD.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.log.txt"
+           curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD.gh.log.txt"
+           curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks-Extras/x86_64-Linux/BUILD_FAILED.gh.log.txt" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BUILD_FAILED.gh.log.txt"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/BLAKE3SUM.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/BLAKE3SUM.json"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/FILE.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/FILE.json"
            curl -qfsSL "https://huggingface.co/datasets/Azathothas/Toolpacks-Extras/resolve/main/x86_64-Linux/METADATA.json" -o "${GITHUB_WORKSPACE}/main/x86_64-Linux/METADATA.json"