Introduce todo.txt annotation to mark integration tests as disabled.

Similar to previous `disabled.txt`, but ensures that fixed integration tests don't stay disabled. Failing tests marked as "to do" will be reported as *skipped*; passing tests marked as "to do" will be reported as *failed*. Set environment variable `METAFIX_DISABLE_TO_DO=true` (or system property `org.metafacture.metafix.disableToDo=true` when run via Gradle) to mimic previous `disabled.txt` behaviour. Analogous to `@MetafixToDo` annotation in unit tests (see #158 and f4a7523).
metafacture · Mar 3, 2022 · 86f7831 · 86f7831
1 parent f8495c2
commit 86f7831
Show file tree

Hide file tree

Showing 52 changed files with 50 additions and 29 deletions.
diff --git a/build.gradle b/build.gradle
@@ -6,14 +6,16 @@ plugins {
 
 editorconfig {
   excludes = [
-    '**/generated',
-    '**/xtext-gen',
-    '**/bin',
+    '**/*.out',
+    '**/*.vsix',
     '**/.*',
-    'gradlew*',
-    '**/out',
+    '**/bin',
+    '**/generated',
     '**/node_modules',
-    '**/*.vsix'
+    '**/out',
+    '**/output-*',
+    '**/xtext-gen',
+    'gradlew*'
   ]
 }
 

diff --git a/metafix/build.gradle b/metafix/build.gradle
@@ -63,7 +63,9 @@ test {
 
 task integrationTest(type: Exec, group: 'Verification') {
   executable './integrationTest.sh'
-  if (project.hasProperty("args")) args project.getProperty("args").split()
+
+  if (project.hasProperty('args')) args project.getProperty('args').split()
+  environment.METAFIX_DISABLE_TO_DO = System.getProperty('org.metafacture.metafix.disableToDo')
 }
 
 task install(dependsOn: publishToMavenLocal, group: 'Publishing',

diff --git a/metafix/integrationTest.sh b/metafix/integrationTest.sh
@@ -8,7 +8,7 @@ metafix_file=test.flux
 catmandu_file=test.cmd
 
 fix_file=test.fix
-disabled_file=disabled.txt
+todo_file=todo.txt
 
 input_glob=input.*
 expected_glob=expected.*
@@ -20,6 +20,12 @@ root_directory="$PWD"
 data_directory="$root_directory/src/test/resources/org/metafacture/metafix/integration"
 gradle_command="$root_directory/../gradlew"
 
+function parse_boolean() {
+  [ "${1,,}" == true ]
+}
+
+parse_boolean "$METAFIX_DISABLE_TO_DO" && disable_todo=1 || disable_todo=
+
 [ -t 1 -a -x /usr/bin/colordiff ] && colordiff=colordiff || colordiff=cat
 
 function _tput() {
@@ -112,13 +118,30 @@ function command_info() {
 
   [ -s "$3" ] && log "  ${color_info}${1^} command output$color_reset: $3" || rm -f "$3"
   [ -s "$4" ] && log "  ${color_info}${1^} command error$color_reset:  $4" || rm -f "$4"
+
+  log
+}
+
+function skip_test() {
+  if [ -r "$2" ]; then
+    local message="$color_test$1$color_reset: ${color_skipped}SKIPPED$color_reset" reason=$(head -1 "$2")
+
+    [ -n "$reason" ] && message+=" ($reason)"
+    log "$message"
+
+    ((skipped++)) || true
+
+    return 0;
+  else
+    return 1;
+  fi
 }
 
 function run_tests() {
   local test matched=1\
-    test_input test_expected test_disabled\
-    metafix_command_output metafix_command_error\
-    metafix_exit_status metafix_output metafix_diff
+    test_directory test_fix test_input test_expected test_todo\
+    metafix_command_output metafix_command_error metafix_start_time\
+    metafix_exit_status metafix_output metafix_diff metafix_elapsed_time
 
   cd "$data_directory"
 
@@ -138,13 +161,9 @@ function run_tests() {
     get_file "$test" expected "$test_directory"/$expected_glob || { log; continue; }
     test_expected=$current_file
 
-    test_disabled="$test_directory/$disabled_file"
+    test_todo="$test_directory/$todo_file"
 
-    if [ -r "$test_disabled" ]; then
-      log "$color_test$test$color_reset: ${color_skipped}SKIPPED$color_reset ($(<"$test_disabled"))"
-
-      ((skipped++)) || true
-    else
+    if [ -z "$disable_todo" ] || ! skip_test "$test" "$test_todo"; then
       metafix_command_output="$test_directory/metafix.out"
       metafix_command_error="$test_directory/metafix.err"
 
@@ -162,14 +181,18 @@ function run_tests() {
           metafix_elapsed_time=$(elapsed_time "$metafix_start_time")
 
           if diff -u "$test_expected" "$metafix_output" >"$metafix_diff"; then
-            #log "$color_test$test$color_reset: ${color_passed}PASSED$color_reset$metafix_elapsed_time"
+            if [ -r "$test_todo" ]; then
+              log "$color_test$test$color_reset: ${color_failed}FAILED$color_reset (Marked as \"to do\", but passed.)"
 
-            rm -f "$metafix_diff" "$metafix_command_output" "$metafix_command_error"
+              ((failed++)) || true
+            else
+              #log "$color_test$test$color_reset: ${color_passed}PASSED$color_reset$metafix_elapsed_time"
 
-            ((passed++)) || true
+              ((passed++)) || true
+            fi
 
-            #log
-          else
+            rm -f "$metafix_diff" "$metafix_command_output" "$metafix_command_error"
+          elif ! skip_test "$test" "$test_todo"; then
             log "$color_test$test$color_reset: ${color_failed}FAILED$color_reset$metafix_elapsed_time"
 
             log "  Fix:      $test_fix"
@@ -183,24 +206,18 @@ function run_tests() {
             command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
 
             ((failed++)) || true
-
-            log
           fi
         else
           command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
-
-          log
         fi
-      else
+      elif ! skip_test "$test" "$test_todo"; then
         metafix_exit_status=$?
 
         log "$color_test$test$color_reset: ${color_error}ERROR$color_reset"
 
         command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
 
         ((failed++)) || true
-
-        log
       fi
     fi
   done

diff --git a/...StringsWithVariableAndAppend/disabled.txt → ...ayOfStringsWithVariableAndAppend/todo.txt b/...StringsWithVariableAndAppend/disabled.txt → ...ayOfStringsWithVariableAndAppend/todo.txt
diff --git a/...eWithInternalMapWithoutMatch/disabled.txt → ...impleWithInternalMapWithoutMatch/todo.txt b/...eWithInternalMapWithoutMatch/disabled.txt → ...impleWithInternalMapWithoutMatch/todo.txt
diff --git a/...bjectsWithFirstArrayWildcard/disabled.txt → ...yOfObjectsWithFirstArrayWildcard/todo.txt b/...bjectsWithFirstArrayWildcard/disabled.txt → ...yOfObjectsWithFirstArrayWildcard/todo.txt
diff --git a/...ObjectsWithLastArrayWildcard/disabled.txt → ...ayOfObjectsWithLastArrayWildcard/todo.txt b/...ObjectsWithLastArrayWildcard/disabled.txt → ...ayOfObjectsWithLastArrayWildcard/todo.txt
diff --git a/...thod/fromJson/toJson/flatten/disabled.txt → ...n/method/fromJson/toJson/flatten/todo.txt b/...thod/fromJson/toJson/flatten/disabled.txt → ...n/method/fromJson/toJson/flatten/todo.txt
diff --git a/...bjectsWithFirstArrayWildcard/disabled.txt → ...yOfObjectsWithFirstArrayWildcard/todo.txt b/...bjectsWithFirstArrayWildcard/disabled.txt → ...yOfObjectsWithFirstArrayWildcard/todo.txt
diff --git a/...ObjectsWithLastArrayWildcard/disabled.txt → ...ayOfObjectsWithLastArrayWildcard/todo.txt b/...ObjectsWithLastArrayWildcard/disabled.txt → ...ayOfObjectsWithLastArrayWildcard/todo.txt
diff --git a/...erseSubfieldInArrayOfObjects/disabled.txt → .../reverseSubfieldInArrayOfObjects/todo.txt b/...erseSubfieldInArrayOfObjects/disabled.txt → .../reverseSubfieldInArrayOfObjects/todo.txt
diff --git a/...everseValuesInArrayOfStrings/disabled.txt → ...on/reverseValuesInArrayOfStrings/todo.txt b/...everseValuesInArrayOfStrings/disabled.txt → ...on/reverseValuesInArrayOfStrings/todo.txt
diff --git a/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ieldInArrayOfObjectsWithAsterisk/todo.txt b/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ieldInArrayOfObjectsWithAsterisk/todo.txt
diff --git a/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ieldInArrayOfObjectsWithAsterisk/todo.txt b/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ieldInArrayOfObjectsWithAsterisk/todo.txt
diff --git a/.../sumSubfieldInArrayOfObjects/disabled.txt → ...Json/sumSubfieldInArrayOfObjects/todo.txt b/.../sumSubfieldInArrayOfObjects/disabled.txt → ...Json/sumSubfieldInArrayOfObjects/todo.txt
diff --git a/...dd_fieldAppendAndCreateArray/disabled.txt → ...on/add_fieldAppendAndCreateArray/todo.txt b/...dd_fieldAppendAndCreateArray/disabled.txt → ...on/add_fieldAppendAndCreateArray/todo.txt
diff --git a/...ldAsteriskIntoArrayOfObjects/disabled.txt → ..._fieldAsteriskIntoArrayOfObjects/todo.txt b/...ldAsteriskIntoArrayOfObjects/disabled.txt → ..._fieldAsteriskIntoArrayOfObjects/todo.txt
diff --git a/...d_fieldPrependAndCreateArray/disabled.txt → ...n/add_fieldPrependAndCreateArray/todo.txt b/...d_fieldPrependAndCreateArray/disabled.txt → ...n/add_fieldPrependAndCreateArray/todo.txt
diff --git a/...n/add_fieldSimpleDestructive/disabled.txt → ...oJson/add_fieldSimpleDestructive/todo.txt b/...n/add_fieldSimpleDestructive/disabled.txt → ...oJson/add_fieldSimpleDestructive/todo.txt
diff --git a/...n/add_fieldSimpleSubsubfield/disabled.txt → ...oJson/add_fieldSimpleSubsubfield/todo.txt b/...n/add_fieldSimpleSubsubfield/disabled.txt → ...oJson/add_fieldSimpleSubsubfield/todo.txt
diff --git a/...py_fieldAppendAndCreateArray/disabled.txt → ...n/copy_fieldAppendAndCreateArray/todo.txt b/...py_fieldAppendAndCreateArray/disabled.txt → ...n/copy_fieldAppendAndCreateArray/todo.txt
diff --git a/...AppendObjectToArrayOfObjects/disabled.txt → ...ieldAppendObjectToArrayOfObjects/todo.txt b/...AppendObjectToArrayOfObjects/disabled.txt → ...ieldAppendObjectToArrayOfObjects/todo.txt
diff --git a/...rayOfObjectsWithLastWildcard/disabled.txt → ...heArrayOfObjectsWithLastWildcard/todo.txt b/...rayOfObjectsWithLastWildcard/disabled.txt → ...heArrayOfObjectsWithLastWildcard/todo.txt
diff --git a/...fObjectsWithAsteriskWildcard/disabled.txt → ...rayOfObjectsWithAsteriskWildcard/todo.txt b/...fObjectsWithAsteriskWildcard/disabled.txt → ...rayOfObjectsWithAsteriskWildcard/todo.txt
diff --git a/...ldAsteriskIntoArrayOfObjects/disabled.txt → ..._fieldAsteriskIntoArrayOfObjects/todo.txt b/...ldAsteriskIntoArrayOfObjects/disabled.txt → ..._fieldAsteriskIntoArrayOfObjects/todo.txt
diff --git a/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ingsInArrayOfObjectsWithAsterisk/todo.txt b/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ingsInArrayOfObjectsWithAsterisk/todo.txt
diff --git a/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ectsInArrayOfObjectsWithAsterisk/todo.txt b/...InArrayOfObjectsWithAsterisk/disabled.txt → ...ectsInArrayOfObjectsWithAsterisk/todo.txt
diff --git a/...ayOfObjectsWithFirstWildcard/disabled.txt → ...mArrayOfObjectsWithFirstWildcard/todo.txt b/...ayOfObjectsWithFirstWildcard/disabled.txt → ...mArrayOfObjectsWithFirstWildcard/todo.txt
diff --git a/...rayOfObjectsWithLastWildcard/disabled.txt → ...omArrayOfObjectsWithLastWildcard/todo.txt b/...rayOfObjectsWithLastWildcard/disabled.txt → ...omArrayOfObjectsWithLastWildcard/todo.txt
diff --git a/...y_fieldPrependAndCreateArray/disabled.txt → .../copy_fieldPrependAndCreateArray/todo.txt b/...y_fieldPrependAndCreateArray/disabled.txt → .../copy_fieldPrependAndCreateArray/todo.txt
diff --git a/...rependObjectToArrayOfObjects/disabled.txt → ...eldPrependObjectToArrayOfObjects/todo.txt b/...rependObjectToArrayOfObjects/disabled.txt → ...eldPrependObjectToArrayOfObjects/todo.txt
diff --git a/.../copy_fieldSimpleDestructive/disabled.txt → ...Json/copy_fieldSimpleDestructive/todo.txt b/.../copy_fieldSimpleDestructive/disabled.txt → ...Json/copy_fieldSimpleDestructive/todo.txt
diff --git a/...uesOfArrayOfStringsIntoArray/disabled.txt → ...dValuesOfArrayOfStringsIntoArray/todo.txt b/...uesOfArrayOfStringsIntoArray/disabled.txt → ...dValuesOfArrayOfStringsIntoArray/todo.txt
diff --git a/...ationOfFieldInArrayOfObjects/disabled.txt → ...ternationOfFieldInArrayOfObjects/todo.txt b/...ationOfFieldInArrayOfObjects/disabled.txt → ...ternationOfFieldInArrayOfObjects/todo.txt
diff --git a/...eldInMultipleArraysOfObjects/disabled.txt → ...OfFieldInMultipleArraysOfObjects/todo.txt b/...eldInMultipleArraysOfObjects/disabled.txt → ...OfFieldInMultipleArraysOfObjects/todo.txt
diff --git a/...ithAlternationOfNestedFields/disabled.txt → ...eldWithAlternationOfNestedFields/todo.txt b/...ithAlternationOfNestedFields/disabled.txt → ...eldWithAlternationOfNestedFields/todo.txt
diff --git a/...ve_fieldAppendAndCreateArray/disabled.txt → ...n/move_fieldAppendAndCreateArray/todo.txt b/...ve_fieldAppendAndCreateArray/disabled.txt → ...n/move_fieldAppendAndCreateArray/todo.txt
diff --git a/...AppendObjectToArrayOfObjects/disabled.txt → ...ieldAppendObjectToArrayOfObjects/todo.txt b/...AppendObjectToArrayOfObjects/disabled.txt → ...ieldAppendObjectToArrayOfObjects/todo.txt
diff --git a/...fStringsWithinArrayOfObjects/disabled.txt → ...rayOfStringsWithinArrayOfObjects/todo.txt b/...fStringsWithinArrayOfObjects/disabled.txt → ...rayOfStringsWithinArrayOfObjects/todo.txt
diff --git a/...ldAsteriskIntoArrayOfObjects/disabled.txt → ..._fieldAsteriskIntoArrayOfObjects/todo.txt b/...ldAsteriskIntoArrayOfObjects/disabled.txt → ..._fieldAsteriskIntoArrayOfObjects/todo.txt
diff --git a/...on/move_fieldIntoOwnSubfield/disabled.txt → ...toJson/move_fieldIntoOwnSubfield/todo.txt b/...on/move_fieldIntoOwnSubfield/disabled.txt → ...toJson/move_fieldIntoOwnSubfield/todo.txt
diff --git a/...e_fieldPrependAndCreateArray/disabled.txt → .../move_fieldPrependAndCreateArray/todo.txt b/...e_fieldPrependAndCreateArray/disabled.txt → .../move_fieldPrependAndCreateArray/todo.txt
diff --git a/...rependObjectToArrayOfObjects/disabled.txt → ...eldPrependObjectToArrayOfObjects/todo.txt b/...rependObjectToArrayOfObjects/disabled.txt → ...eldPrependObjectToArrayOfObjects/todo.txt
diff --git a/.../move_fieldSimpleDestructive/disabled.txt → ...Json/move_fieldSimpleDestructive/todo.txt b/.../move_fieldSimpleDestructive/disabled.txt → ...Json/move_fieldSimpleDestructive/todo.txt
diff --git a/...mpleWithAsteriskPathWildcard/disabled.txt → ...ldSimpleWithAsteriskPathWildcard/todo.txt b/...mpleWithAsteriskPathWildcard/disabled.txt → ...ldSimpleWithAsteriskPathWildcard/todo.txt
diff --git a/...dingQuestionmarkPathWildcard/disabled.txt → ...hLeadingQuestionmarkPathWildcard/todo.txt b/...dingQuestionmarkPathWildcard/disabled.txt → ...hLeadingQuestionmarkPathWildcard/todo.txt
diff --git a/...TrailingAsteriskPathWildcard/disabled.txt → ...WithTrailingAsteriskPathWildcard/todo.txt b/...TrailingAsteriskPathWildcard/disabled.txt → ...WithTrailingAsteriskPathWildcard/todo.txt
diff --git a/...ationOfFieldInArrayOfObjects/disabled.txt → ...ternationOfFieldInArrayOfObjects/todo.txt b/...ationOfFieldInArrayOfObjects/disabled.txt → ...ternationOfFieldInArrayOfObjects/todo.txt
diff --git a/...eldInMultipleArraysOfObjects/disabled.txt → ...OfFieldInMultipleArraysOfObjects/todo.txt b/...eldInMultipleArraysOfObjects/disabled.txt → ...OfFieldInMultipleArraysOfObjects/todo.txt
diff --git a/...ithAlternationOfNestedFields/disabled.txt → ...eldWithAlternationOfNestedFields/todo.txt b/...ithAlternationOfNestedFields/disabled.txt → ...eldWithAlternationOfNestedFields/todo.txt
diff --git a/...oJson/retainCertainSubfields/disabled.txt → ...on/toJson/retainCertainSubfields/todo.txt b/...oJson/retainCertainSubfields/disabled.txt → ...on/toJson/retainCertainSubfields/todo.txt
diff --git a/...on/toJson/set_arrayIntoArray/disabled.txt → ...omJson/toJson/set_arrayIntoArray/todo.txt b/...on/toJson/set_arrayIntoArray/disabled.txt → ...omJson/toJson/set_arrayIntoArray/todo.txt