tests: rename skip_on_jobname to expect_skip_on_jobname and skip_on_o…

…s to expect_skip_on_os

The test.json format currently has three keys related to skipping tests:

* `skip_on_jobname`
* `skip_on_os`
* `skip_on_env`

While `skip_on_env` marks the test itself as skipped, i.e. they don't get run when the
conditions are met, the other two skip options are just marking the test as "expected to be skipped"
if the conditions apply, i.e. they want to see `MESON_SKIP_TEST` in the output and things will
fail if that doesn't happen. They don't actually skip the tests as the names imply.

To make this clearer rename the keys:

* `skip_on_jobname` -> `expect_skip_on_jobname`
* `skip_on_os` -> `expect_skip_on_os`

`skip_on_env` stays the same, since that actually skips.

The docs were also confused about this, so adjust that too.

data/test.schema.json

@@ -87,13 +87,13 @@
                     "type": "string"
                   }
                 },
-                "skip_on_jobname": {
+                "expect_skip_on_jobname": {
                   "type": "array",
                   "items": {
                     "type": "string"
                   }
                 },
-                "skip_on_os": {
+                "expect_skip_on_os": {
                   "type": "array",
                   "items": {
                     "type": "string"
@@ -167,13 +167,13 @@
         "type": "string"
       }
     },
-    "skip_on_jobname": {
+    "expect_skip_on_jobname": {
       "type": "array",
       "items": {
         "type": "string"
       }
     },
-    "skip_on_os": {
+    "expect_skip_on_os": {
       "type": "array",
       "items": {
         "type": "string"

docs/markdown/Contributing.md

@@ -334,9 +334,11 @@ Each value must contain the `val` key for the value of the option.
 `null` can be used for adding matrix entries without the current
 option.
 
-The `skip_on_env`, `skip_on_jobname` and `skip_on_os` keys (as described below)
-may be used in the value to skip that matrix entry, based on the current
-environment.
+The `skip_on_env` key (as described below) may be used in the value to skip that
+matrix entry, based on the current environment.
+
+The `expect_skip_on_jobname` and `expect_skip_on_os` keys (as described below)
+may be used to expect that the test will be skipped, based on the current environment.
 
 Similarly, the `compilers` key can be used to define a mapping of
 compilers to languages that are required for this value.
@@ -412,23 +414,23 @@ The `skip_on_env` key can be used to specify a list of environment variables. If
 at least one environment variable in the `skip_on_env` list is present, the test
 is skipped.
 
-#### skip_on_jobname
+#### expect_skip_on_jobname
 
-The `skip_on_jobname` key contains a list of strings. If the `MESON_CI_JOBNAME`
+The `expect_skip_on_jobname` key contains a list of strings. If the `MESON_CI_JOBNAME`
 environment variable is set, and any of them are a sub-string of it, the test is
 expected to be skipped (that is, it is expected that the test will output
 `MESON_SKIP_TEST`, because the CI environment is not one in which it can run,
 for whatever reason).
 
-The test is failed if it skips or runs unexpectedly.
+The test is failed if it either skips unexpectedly or runs unexpectedly.
 
-#### skip_on_os
+#### expect_skip_on_os
 
-The `skip_on_os` key can be used to specify a list of OS names (or their
-negations, prefixed with a `!`).  If at least one item in the `skip_on_os` list
+The `expect_skip_on_os` key can be used to specify a list of OS names (or their
+negations, prefixed with a `!`).  If at least one item in the `expect_skip_on_os` list
 is matched, the test is expected to be skipped.
 
-The test is failed if it skips or runs unexpectedly.
+The test is failed if it either skips unexpectedly or runs unexpectedly.
 
 ### Skipping integration tests
 

run_project_tests.py

@@ -785,13 +785,13 @@ def _skip_keys(test_def: T.Dict) -> T.Tuple[bool, bool]:
 
     # Test is expected to skip if MESON_CI_JOBNAME contains any of the list of
     # substrings
-    if ('skip_on_jobname' in test_def) and (ci_jobname is not None):
-        skip_expected = any(s in ci_jobname for s in test_def['skip_on_jobname'])
+    if ('expect_skip_on_jobname' in test_def) and (ci_jobname is not None):
+        skip_expected = any(s in ci_jobname for s in test_def['expect_skip_on_jobname'])
 
     # Test is expected to skip if os matches
-    if 'skip_on_os' in test_def:
+    if 'expect_skip_on_os' in test_def:
         mesonenv = environment.Environment('', '', get_fake_options('/'))
-        for skip_os in test_def['skip_on_os']:
+        for skip_os in test_def['expect_skip_on_os']:
             if skip_os.startswith('!'):
                 if mesonenv.machines.host.system != skip_os[1:]:
                     skip_expected = True

test cases/frameworks/1 boost/test.json

@@ -18,5 +18,5 @@
       { "static": "false", "b_vscrt": "mtd" }
     ]
   },
-  "skip_on_jobname": ["azure", "msys2"]
+  "expect_skip_on_jobname": ["azure", "msys2"]
 }

test cases/frameworks/10 gtk-doc/test.json

@@ -60,5 +60,5 @@
     {"type": "file", "file": "usr/share/gtk-doc/html/foobar3/up.png"},
     {"type": "file", "file": "usr/share/gtk-doc/html/foobar3/up-insensitive.png"}
   ],
-  "skip_on_jobname": ["azure", "msys2"]
+  "expect_skip_on_jobname": ["azure", "msys2"]
 }

test cases/frameworks/11 gir subproject/test.json

@@ -9,5 +9,5 @@
     {"type": "expr", "file": "usr/lib/?libgirlib.so"},
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirsubproject.dll.a"}
   ],
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"]
 }

test cases/frameworks/12 multiple gir/test.json

@@ -7,7 +7,7 @@
     {"type": "expr", "file": "usr/lib/?libgirsubproject.so"},
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirsubproject.dll.a"}
   ],
-  "skip_on_jobname": ["azure", "macos", "msys2"],
+  "expect_skip_on_jobname": ["azure", "macos", "msys2"],
   "stdout": [
     {
       "comment": "This will either match in the future-deprecated notice summary, or match the warning summary",

test cases/frameworks/13 yelp/test.json

@@ -19,5 +19,5 @@
     {"type": "file", "file": "usr/share/help/de/meson-linguas/index3.page"},
     {"type": "file", "file": "usr/share/help/de/meson-linguas/media/test.txt"}
   ],
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/14 doxygen/test.json

@@ -2,5 +2,5 @@
   "installed": [
     {"type": "dir", "file": "usr/share/doc/spede/html"}
   ],
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/15 llvm/test.json

@@ -2,17 +2,17 @@
   "matrix": {
     "options": {
       "method": [
-        { "val": "config-tool", "skip_on_jobname": ["msys2-gcc"] },
-        { "val": "cmake", "skip_on_jobname": ["msys2"] },
-        { "val": "combination", "skip_on_jobname": ["msys2"] }
+        { "val": "config-tool", "expect_skip_on_jobname": ["msys2-gcc"] },
+        { "val": "cmake", "expect_skip_on_jobname": ["msys2"] },
+        { "val": "combination", "expect_skip_on_jobname": ["msys2"] }
       ],
       "link-static": [
-        { "val": true, "skip_on_jobname": ["arch", "opensuse", "linux-gentoo-gcc"] },
+        { "val": true, "expect_skip_on_jobname": ["arch", "opensuse", "linux-gentoo-gcc"] },
         { "val": false }
       ]
     }
   },
-  "skip_on_jobname": ["azure", "cygwin"],
+  "expect_skip_on_jobname": ["azure", "cygwin"],
   "tools": {
     "cmake": ">=3.11"
   }

test cases/frameworks/16 sdl2/test.json

@@ -6,10 +6,10 @@
         { "val": "pkg-config" },
         { "val": "config-tool" },
         { "val": "sdlconfig" },
-        { "val": "extraframework", "skip_on_os": ["!darwin"], "skip_on_jobname": ["macos"] },
-        { "val": "cmake", "skip_on_jobname": ["bionic"] }
+        { "val": "extraframework", "expect_skip_on_os": ["!darwin"], "expect_skip_on_jobname": ["macos"] },
+        { "val": "cmake", "expect_skip_on_jobname": ["bionic"] }
       ]
     }
   },
-  "skip_on_jobname": ["azure", "cygwin", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "msys2"]
 }

test cases/frameworks/17 mpi/test.json

@@ -5,13 +5,13 @@
         { "val": "auto" },
         { "val": "pkg-config" },
         { "val": "config-tool",
-          "skip_on_jobname": ["fedora"] },
+          "expect_skip_on_jobname": ["fedora"] },
         {
           "val": "system",
           "compilers": { "c" :"msvc", "cpp": "msvc" }
         }
       ]
     }
   },
-  "skip_on_jobname": ["azure", "cygwin", "msys2", "opensuse"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "msys2", "opensuse"]
 }

test cases/frameworks/18 vulkan/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/19 pcap/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "msys2"]
 }

test cases/frameworks/2 gtest/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/20 cups/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "msys2"]
 }

test cases/frameworks/21 libwmf/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/22 gir link order/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "macos", "msys2"]
 }

test cases/frameworks/23 hotdoc/test.json

@@ -5,5 +5,5 @@
   "tools": {
     "hotdoc": ">=0.1.0"
   },
-  "skip_on_jobname": ["msys2"]
+  "expect_skip_on_jobname": ["msys2"]
 }

test cases/frameworks/24 libgcrypt/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "msys2"]
+  "expect_skip_on_jobname": ["azure", "msys2"]
 }

test cases/frameworks/25 hdf5/test.json

@@ -2,10 +2,10 @@
   "matrix": {
     "options": {
       "method": [
-        { "val": "pkg-config", "skip_on_jobname": ["linux-gentoo-gcc"] },
-        { "val": "config-tool", "skip_on_jobname": ["macos"] }
+        { "val": "pkg-config", "expect_skip_on_jobname": ["linux-gentoo-gcc"] },
+        { "val": "config-tool", "expect_skip_on_jobname": ["macos"] }
       ]
     }
   },
-  "skip_on_jobname": ["azure", "cygwin", "fedora", "msys2", "opensuse"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "fedora", "msys2", "opensuse"]
 }

test cases/frameworks/26 netcdf/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "macos", "msys2", "opensuse", "ubuntu"]
+  "expect_skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "macos", "msys2", "opensuse", "ubuntu"]
 }

test cases/frameworks/27 gpgme/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "linux-arch", "ubuntu"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "linux-arch", "ubuntu"]
 }

test cases/frameworks/28 gir link order 2/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "macos", "msys2"]
 }

test cases/frameworks/29 blocks/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "gcc", "msys2"]
+  "expect_skip_on_jobname": ["azure", "gcc", "msys2"]
 }

test cases/frameworks/3 gmock/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/30 scalapack/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "msys2", "opensuse"]
+  "expect_skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "msys2", "opensuse"]
 }

test cases/frameworks/31 curses/test.json

@@ -3,10 +3,10 @@
     "options": {
       "method": [
         { "val": "pkg-config" },
-        { "val": "config-tool", "skip_on_jobname": ["msys2"] },
-        { "val": "system", "skip_on_os": ["windows"] }
+        { "val": "config-tool", "expect_skip_on_jobname": ["msys2"] },
+        { "val": "system", "expect_skip_on_os": ["windows"] }
       ]
     }
   },
-  "skip_on_jobname": ["azure", "cygwin"]
+  "expect_skip_on_jobname": ["azure", "cygwin"]
 }

test cases/frameworks/34 gir static lib/test.json

@@ -5,5 +5,5 @@
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirlib.dll.a"},
     {"type": "file", "file": "usr/share/gir-1.0/Meson-1.0.gir"}
   ],
-  "skip_on_jobname": ["azure", "bionic", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "bionic", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/36 gtkdoc cpp/test.json

@@ -13,5 +13,5 @@
     {"type": "file", "file": "usr/share/gtk-doc/html/foo/up.png"},
     {"type": "file", "file": "usr/share/gtk-doc/html/foo/right.png"}
   ],
-  "skip_on_jobname": ["azure", "msys2"]
+  "expect_skip_on_jobname": ["azure", "msys2"]
 }

test cases/frameworks/37 gir cpp/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "macos", "msys2", "cygwin"]
+  "expect_skip_on_jobname": ["azure", "macos", "msys2", "cygwin"]
 }

test cases/frameworks/4 qt/test.json

@@ -8,5 +8,5 @@
       ]
     }
   },
-  "skip_on_jobname": ["cygwin", "msys2", "azure"]
+  "expect_skip_on_jobname": ["cygwin", "msys2", "azure"]
 }

test cases/frameworks/5 protocol buffers/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/6 gettext/test.json

@@ -22,5 +22,5 @@
       ]
     }
   },
-  "skip_on_jobname": ["azure", "cygwin"]
+  "expect_skip_on_jobname": ["azure", "cygwin"]
 }

test cases/frameworks/7 gnome/test.json

@@ -36,5 +36,5 @@
     {"type": "file", "file": "usr/include/simple-resources.h"},
     {"type": "file", "file": "usr/include/generated-gdbus.h"}
   ],
-  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"]
+  "expect_skip_on_jobname": ["azure", "cygwin", "macos", "msys2", "pypy"]
 }

test cases/frameworks/8 flex/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["azure", "cygwin"]
+  "expect_skip_on_jobname": ["azure", "cygwin"]
 }