From a3094e3fab977edba5af671265d8894426fc8702 Mon Sep 17 00:00:00 2001 From: yuunlimm Date: Mon, 7 Oct 2024 16:31:40 -0700 Subject: [PATCH] convert to bash script for readability --- .../workflows/indexer-processor-testing.yaml | 223 +++++++++--------- scripts/indexer_test_txns_compare_and_diff.sh | 108 +++++++++ 2 files changed, 220 insertions(+), 111 deletions(-) create mode 100644 scripts/indexer_test_txns_compare_and_diff.sh diff --git a/.github/workflows/indexer-processor-testing.yaml b/.github/workflows/indexer-processor-testing.yaml index 40757aefe05bb..00c069136a99f 100644 --- a/.github/workflows/indexer-processor-testing.yaml +++ b/.github/workflows/indexer-processor-testing.yaml @@ -50,7 +50,7 @@ jobs: - name: Prepare and Clean JSON Files run: | cd ecosystem/indexer-grpc/indexer-test-transactions - + for folder in json_transactions/scripted_transactions new_json_transactions/scripted_transactions; do for file in $folder/*.json; do echo "Processing $file..." @@ -87,116 +87,117 @@ jobs: - name: Compare JSON Files Across Multiple Folders id: diff_check run: | - cd ecosystem/indexer-grpc/indexer-test-transactions - - # Define a function to compare and handle diff logic - compare_and_diff() { - local generated_file=$1 - local original_file=$2 - - echo "Modified file path: $generated_file" - echo "Original file path: $original_file" - - if [ -f "$original_file" ]; then - echo "Original file exists, comparing with modified file." - # Run diff and capture the output - diff_output=$(diff -u "$original_file" "$generated_file" || true) - - if [ -n "$diff_output" ]; then - echo "Differences found in $generated_file" - diff_found=true - modified_files="${modified_files}${generated_file}\n" # Append the full path of the modified file - echo "Diff output:" - echo "$diff_output" - else - echo "No differences found for $generated_file." - fi - else - echo "New file detected: $generated_file (no corresponding original file found)" - new_file_found=true - new_files="${new_files}${generated_file}\n" # Append the full path of the new file - - # Treat as new file, but still run a diff (compare with /dev/null) - diff_output=$(diff -u /dev/null "$generated_file" || true) - if [ -n "$diff_output" ]; then - echo "New file with diff found in $generated_file" - diff_found=true - echo "Diff output for new file:" - echo "$diff_output" - fi - fi - } - - - # Initialize the flags - diff_found=false - new_file_found=false - new_files="" - modified_files="" - - echo "Starting comparison between new and original JSON files." - - # Check if the new_json_transactions folder exists - if [ ! -d "new_json_transactions" ]; then - echo "Directory new_json_transactions does not exist. Exiting." - exit 1 - fi - - # Loop over all subdirectories inside new_json_transactions - for folder in new_json_transactions/*; do - if [ -d "$folder" ]; then # Ensure it's a directory - echo "Processing folder: $folder" - - # Check if the folder is for imported transactions - if [[ "$folder" == *"imported_"* ]]; then - # For imported transactions, process all files without any 'modified_' check - for file in "$folder"/*.json; do - if [ -f "$file" ]; then - echo "Processing imported file: $file" - base_file=$(basename "$file" .json) - original_file="../indexer-test-transactions/json_transactions/$(basename $folder)/${base_file}.json" - compare_and_diff "$file" "$original_file" - fi - done - else - # For scripted transactions, only process files that are prefixed with 'cleaned_' - for file in "$folder"/cleaned_*.json; do - if [ -f "$file" ]; then - echo "Processing scripted file: $file" - base_file=$(basename "$file" .json) - original_file="../indexer-test-transactions/json_transactions/$(basename $folder)/${base_file}.json" - compare_and_diff "$file" "$original_file" - fi - done - fi - else - echo "Folder $folder is not a valid directory." - fi - done - - # Print all new files if found - if [ "$new_file_found" = "true" ] && [ -n "$new_files" ]; then - echo "New files detected:" - echo -e "$new_files" - else - echo "No new files detected." - fi - - # Print all modified files if found - if [ "$diff_found" = "true" ] && [ -n "$modified_files" ]; then - echo "Modified files detected:" - echo -e "$modified_files" - else - echo "No modified files detected." - fi - - # Set output flags - echo "Setting output flags..." - echo "diff_found=$diff_found" >> $GITHUB_OUTPUT - echo "new_file_found=$new_file_found" >> $GITHUB_OUTPUT - echo "new_files=$new_files" >> $GITHUB_OUTPUT # Store new files as output - echo "modified_files=$modified_files" >> $GITHUB_OUTPUT # Store modified files as output - echo "Comparison completed." + . scripts/indexer_test_txns_compare_and_diff.sh + # cd ecosystem/indexer-grpc/indexer-test-transactions +# +# # Define a function to compare and handle diff logic +# compare_and_diff() { +# local generated_file=$1 +# local original_file=$2 +# +# echo "Modified file path: $generated_file" +# echo "Original file path: $original_file" +# +# if [ -f "$original_file" ]; then +# echo "Original file exists, comparing with modified file." +# # Run diff and capture the output +# diff_output=$(diff -u "$original_file" "$generated_file" || true) +# +# if [ -n "$diff_output" ]; then +# echo "Differences found in $generated_file" +# diff_found=true +# modified_files="${modified_files}${generated_file}\n" # Append the full path of the modified file +# echo "Diff output:" +# echo "$diff_output" +# else +# echo "No differences found for $generated_file." +# fi +# else +# echo "New file detected: $generated_file (no corresponding original file found)" +# new_file_found=true +# new_files="${new_files}${generated_file}\n" # Append the full path of the new file +# +# # Treat as new file, but still run a diff (compare with /dev/null) +# diff_output=$(diff -u /dev/null "$generated_file" || true) +# if [ -n "$diff_output" ]; then +# echo "New file with diff found in $generated_file" +# diff_found=true +# echo "Diff output for new file:" +# echo "$diff_output" +# fi +# fi +# } +# +# +# # Initialize the flags +# diff_found=false +# new_file_found=false +# new_files="" +# modified_files="" +# +# echo "Starting comparison between new and original JSON files." +# +# # Check if the new_json_transactions folder exists +# if [ ! -d "new_json_transactions" ]; then +# echo "Directory new_json_transactions does not exist. Exiting." +# exit 1 +# fi +# +# # Loop over all subdirectories inside new_json_transactions +# for folder in new_json_transactions/*; do +# if [ -d "$folder" ]; then # Ensure it's a directory +# echo "Processing folder: $folder" +# +# # Check if the folder is for imported transactions +# if [[ "$folder" == *"imported_"* ]]; then +# # For imported transactions, process all files without any 'modified_' check +# for file in "$folder"/*.json; do +# if [ -f "$file" ]; then +# echo "Processing imported file: $file" +# base_file=$(basename "$file" .json) +# original_file="../indexer-test-transactions/json_transactions/$(basename $folder)/${base_file}.json" +# compare_and_diff "$file" "$original_file" +# fi +# done +# else +# # For scripted transactions, only process files that are prefixed with 'cleaned_' +# for file in "$folder"/cleaned_*.json; do +# if [ -f "$file" ]; then +# echo "Processing scripted file: $file" +# base_file=$(basename "$file" .json) +# original_file="../indexer-test-transactions/json_transactions/$(basename $folder)/${base_file}.json" +# compare_and_diff "$file" "$original_file" +# fi +# done +# fi +# else +# echo "Folder $folder is not a valid directory." +# fi +# done +# +# # Print all new files if found +# if [ "$new_file_found" = "true" ] && [ -n "$new_files" ]; then +# echo "New files detected:" +# echo -e "$new_files" +# else +# echo "No new files detected." +# fi +# +# # Print all modified files if found +# if [ "$diff_found" = "true" ] && [ -n "$modified_files" ]; then +# echo "Modified files detected:" +# echo -e "$modified_files" +# else +# echo "No modified files detected." +# fi +# +# # Set output flags +# echo "Setting output flags..." +# echo "diff_found=$diff_found" >> $GITHUB_OUTPUT +# echo "new_file_found=$new_file_found" >> $GITHUB_OUTPUT +# echo "new_files=$new_files" >> $GITHUB_OUTPUT # Store new files as output +# echo "modified_files=$modified_files" >> $GITHUB_OUTPUT # Store modified files as output +# echo "Comparison completed." - name: Handle New Files and Differences diff --git a/scripts/indexer_test_txns_compare_and_diff.sh b/scripts/indexer_test_txns_compare_and_diff.sh new file mode 100644 index 0000000000000..c9922d9ea49d1 --- /dev/null +++ b/scripts/indexer_test_txns_compare_and_diff.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# Function to compare and handle diff logic +compare_and_diff() { + local generated_file=$1 + local original_file=$2 + + echo "Modified file path: $generated_file" + echo "Original file path: $original_file" + + if [ -f "$original_file" ]; then + echo "Original file exists, comparing with modified file." + # Run diff and capture the output + diff_output=$(diff -u "$original_file" "$generated_file" || true) + + if [ -n "$diff_output" ]; then + echo "Differences found in $generated_file" + diff_found=true + modified_files="${modified_files}${generated_file}\n" # Append the full path of the modified file + echo "Diff output:" + echo "$diff_output" + else + echo "No differences found for $generated_file." + fi + else + echo "New file detected: $generated_file (no corresponding original file found)" + new_file_found=true + new_files="${new_files}${generated_file}\n" # Append the full path of the new file + + # Treat as new file, but still run a diff (compare with /dev/null) + diff_output=$(diff -u /dev/null "$generated_file" || true) + if [ -n "$diff_output" ]; then + echo "New file with diff found in $generated_file" + diff_found=true + echo "Diff output for new file:" + echo "$diff_output" + fi + fi +} + +# Initialize the flags +diff_found=false +new_file_found=false +new_files="" +modified_files="" + +echo "Starting comparison between new and original JSON files." + +# Check if the new_json_transactions folder exists +if [ ! -d "new_json_transactions" ]; then + echo "Directory new_json_transactions does not exist. Exiting." + exit 1 +fi + +# Loop over all subdirectories inside new_json_transactions +for folder in new_json_transactions/*; do + if [ -d "$folder" ]; then # Ensure it's a directory + echo "Processing folder: $folder" + + # Check if the folder is for imported transactions + if [[ "$folder" == *"imported_"* ]]; then + # For imported transactions, process all files without any 'modified_' check + for file in "$folder"/*.json; do + if [ -f "$file" ]; then + echo "Processing imported file: $file" + base_file=$(basename "$file" .json) + original_file="../indexer-test-transactions/json_transactions/$(basename $folder)/${base_file}.json" + compare_and_diff "$file" "$original_file" + fi + done + else + # For scripted transactions, only process files that are prefixed with 'cleaned_' + for file in "$folder"/cleaned_*.json; do + if [ -f "$file" ]; then + echo "Processing scripted file: $file" + base_file=$(basename "$file" .json) + original_file="../indexer-test-transactions/json_transactions/$(basename $folder)/${base_file}.json" + compare_and_diff "$file" "$original_file" + fi + done + fi + else + echo "Folder $folder is not a valid directory." + fi +done + +# Print all new files if found +if [ "$new_file_found" = "true" ] && [ -n "$new_files" ]; then + echo "New files detected:" + echo -e "$new_files" +else + echo "No new files detected." +fi + +# Print all modified files if found +if [ "$diff_found" = "true" ] && [ -n "$modified_files" ]; then + echo "Modified files detected:" + echo -e "$modified_files" +else + echo "No modified files detected." +fi + +# Set output flags (if integrated in a larger workflow, e.g., GitHub Actions) +echo "diff_found=$diff_found" >> $GITHUB_OUTPUT +echo "new_file_found=$new_file_found" >> $GITHUB_OUTPUT +echo "new_files=$new_files" >> $GITHUB_OUTPUT # Store new files as output +echo "modified_files=$modified_files" >> $GITHUB_OUTPUT # Store modified files as output +echo "Comparison completed."