From 3a1176e137e05156581a131b6456760b4112e0f8 Mon Sep 17 00:00:00 2001
From: Michael Bien <mbien42@gmail.com>
Date: Tue, 4 Jul 2023 22:40:04 +0200
Subject: [PATCH] stabilize unreliable tests for smoother CI operation.

wrap platform/core.windows, favorites and settings into retry script.

mitigates the following exception which causes the tests to be very
unreliable in CI:
java.lang.AssertionError: Should have found a nonproblematic provider
of needs org.openide.modules.jre.JavaFX among [...]

Separate reliable from unreliable PHP tests and wrap unreliable tests
in retry script.
---
 .github/workflows/main.yml                  | 12 ++++++++----
 php/php.editor/nbproject/project.properties | 10 ++++++++--
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index dea7036ff8ff..d5a50bd24118 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -934,7 +934,7 @@ jobs:
         run: ant $OPTS -Dvanilla.javac.exists=true -f platform/core.ui test
 
       - name: platform/core.windows
-        run: ant $OPTS -Dvanilla.javac.exists=true -f platform/core.windows test
+        run: .github/retry.sh ant $OPTS -Dvanilla.javac.exists=true -f platform/core.windows test
 
       - name: platform/editor.mimelookup
         run: ant $OPTS -Dvanilla.javac.exists=true -f platform/editor.mimelookup test
@@ -943,7 +943,7 @@ jobs:
         run: ant $OPTS -Dvanilla.javac.exists=true -f platform/editor.mimelookup.impl test
 
       - name: platform/favorites
-        run: ant $OPTS -Dvanilla.javac.exists=true -f platform/favorites test
+        run: .github/retry.sh ant $OPTS -Dvanilla.javac.exists=true -f platform/favorites test
 
       - name: platform/javahelp
         run: ant $OPTS -Dvanilla.javac.exists=true -f platform/javahelp test-unit
@@ -1143,7 +1143,7 @@ jobs:
         run: ant $OPTS -Dvanilla.javac.exists=true -f platform/sendopts test
 
       - name: platform/settings
-        run: ant $OPTS -Dvanilla.javac.exists=true -f platform/settings test
+        run: .github/retry.sh ant $OPTS -Dvanilla.javac.exists=true -f platform/settings test
 
       - name: platform/spi.actions
         run: ant $OPTS -Dvanilla.javac.exists=true -f platform/spi.actions test
@@ -2234,7 +2234,11 @@ jobs:
       # longest step (~40min)
       - name: php.editor
         if: env.test_php == 'true' && success()
-        run: ant $OPTS -f php/php.editor test
+        run: ant $OPTS -Dtest.config=stable -f php/php.editor test
+
+      - name: php.editor (unreliable tests)
+        if: env.test_php == 'true' && success()
+        run: .github/retry.sh ant $OPTS -Dtest.config=unreliable -f php/php.editor test
 
       - name: php.latte
         run: ant $OPTS -f php/php.latte test
diff --git a/php/php.editor/nbproject/project.properties b/php/php.editor/nbproject/project.properties
index 484326a203e6..0c50d7238700 100644
--- a/php/php.editor/nbproject/project.properties
+++ b/php/php.editor/nbproject/project.properties
@@ -30,9 +30,15 @@ test.config.uicommit.includes=\
 #  org/netbeans/test/php/Commit.class,\
 #  org/netbeans/test/php/project/test*.class
 
-test.config.stable.includes=\
-  org/netbeans/test/php/*/test*.class
+# known to fail sporadically, should be tested separately in CI, ideally with retry script until stabilized
+test.config.unreliable.includes=\
+  org/netbeans/modules/php/editor/verification/IntroduceSuggestionTest.class,\
+  org/netbeans/modules/php/editor/csl/GotoDeclarationPHP81Test.class,\
+  org/netbeans/modules/php/editor/csl/FoldingTest.class
+test.config.unreliable.excludes=
 
+test.config.stable.includes=**/*Test.class
+test.config.stable.excludes=${test.config.unreliable.includes}
 
 test.config.normal.includes=\
   org/netbeans/test/php/Commit.class,\