test(backport): Use pytest.raises match argument to check Error messa…

…ges (#2165)

* Backport PR #2162

tests/test_import.py

@@ -26,9 +26,8 @@ def test_dedupe_parameters():
     ]
     assert len(pyhf.readxml.dedupe_parameters(parameters)) == 1
     parameters[1]['bounds'] = [[0.0, 2.0]]
-    with pytest.raises(RuntimeError) as excinfo:
+    with pytest.raises(RuntimeError, match="SigXsecOverSM"):
         pyhf.readxml.dedupe_parameters(parameters)
-        assert 'SigXsecOverSM' in str(excinfo.value)
 
 
 def test_process_normfactor_configs():
@@ -486,9 +485,10 @@ def test_import_noChannelData(mocker, datadir):
     mocker.patch('pyhf.readxml.import_root_histogram', return_value=(_data, _err))
 
     basedir = datadir.joinpath("xmlimport_noChannelData")
-    with pytest.raises(RuntimeError) as excinfo:
+    with pytest.raises(
+        RuntimeError, match="Channel channel1 is missing data. See issue #1911"
+    ):
         pyhf.readxml.parse(basedir.joinpath("config/example.xml"), basedir)
-        assert 'Channel channel1 is missing data. See issue #1911' in str(excinfo.value)
 
 
 def test_import_missingPOI(mocker, datadir):
@@ -497,11 +497,10 @@ def test_import_missingPOI(mocker, datadir):
     mocker.patch('pyhf.readxml.import_root_histogram', return_value=(_data, _err))
 
     basedir = datadir.joinpath("xmlimport_missingPOI")
-    with pytest.raises(RuntimeError) as excinfo:
+    with pytest.raises(
+        RuntimeError, match="Measurement GaussExample is missing POI specification"
+    ):
         pyhf.readxml.parse(basedir.joinpath("config/example.xml"), basedir)
-        assert 'Measurement GaussExample is missing POI specification' in str(
-            excinfo.value
-        )
 
 
 def test_import_resolver():

tests/test_optim.py

@@ -218,9 +218,10 @@ def test_optimizer_unsupported_minimizer_options(optimizer):
     pyhf.set_backend(pyhf.default_backend, optimizer())
     m = pyhf.simplemodels.uncorrelated_background([5.0], [10.0], [3.5])
     data = pyhf.tensorlib.astensor([10.0] + m.config.auxdata)
-    with pytest.raises(pyhf.exceptions.Unsupported) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.Unsupported, match="unsupported_minimizer_options"
+    ):
         pyhf.infer.mle.fit(data, m, unsupported_minimizer_options=False)
-    assert 'unsupported_minimizer_options' in str(excinfo.value)
 
 
 @pytest.mark.parametrize('return_result_obj', [False, True], ids=['no_obj', 'obj'])
@@ -437,18 +438,19 @@ def fmin(self):
     pdf = pyhf.simplemodels.uncorrelated_background([5], [10], [3.5])
     data = [10] + pdf.config.auxdata
     spy = mocker.spy(pyhf.optimize.minuit_optimizer, '_minimize')
-    with pytest.raises(pyhf.exceptions.FailedMinimization) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.FailedMinimization, match="Optimization failed"
+    ) as exc_info:
         pyhf.infer.mle.fit(data, pdf)
 
-    assert isinstance(excinfo.value.result, OptimizeResult)
+    assert isinstance(exc_info.value.result, OptimizeResult)
 
-    assert excinfo.match('Optimization failed')
     assert 'Optimization failed' in spy.spy_return.message
     if has_reached_call_limit:
-        assert excinfo.match('Call limit was reached')
+        assert exc_info.match('Call limit was reached')
         assert 'Call limit was reached' in spy.spy_return.message
     if is_above_max_edm:
-        assert excinfo.match('Estimated distance to minimum too large')
+        assert exc_info.match('Estimated distance to minimum too large')
         assert 'Estimated distance to minimum too large' in spy.spy_return.message
 
 
@@ -466,11 +468,9 @@ def test_minuit_set_options(mocker):
 
 def test_get_tensor_shim(monkeypatch):
     monkeypatch.setattr(pyhf.tensorlib, 'name', 'fake_backend')
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(ValueError, match="No optimizer shim for fake_backend."):
         _get_tensor_shim()
 
-    assert 'No optimizer shim for fake_backend.' == str(excinfo.value)
-
 
 def test_stitch_pars(backend):
     tb, _ = backend

tests/test_patchset.py

@@ -74,10 +74,12 @@ def test_patchset_get_patch_by_values(patchset):
 
 
 def test_patchset_get_nonexisting_patch(patchset):
-    with pytest.raises(pyhf.exceptions.InvalidPatchLookup) as excinfo:
+    # Using asserts with str(exc_info.value) over pytest.raises(..., match="...")
+    # to make it easier to check multiple strings.
+    with pytest.raises(pyhf.exceptions.InvalidPatchLookup) as exc_info:
         patchset.__getitem__('nonexisting_patch')
-    assert 'No patch associated with' in str(excinfo.value)
-    assert 'nonexisting_patch' in str(excinfo.value)
+    assert 'No patch associated with' in str(exc_info.value)
+    assert 'nonexisting_patch' in str(exc_info.value)
 
 
 def test_patchset_iterable(patchset):

tests/test_teststats.py

@@ -118,50 +118,45 @@ def test_no_poi_test_stats():
     par_bounds = model.config.suggested_bounds()
     fixed_params = model.config.suggested_fixed()
 
-    with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.UnspecifiedPOI,
+        match="No POI is defined. A POI is required for profile likelihood based test statistics.",
+    ):
         pyhf.infer.test_statistics.q0(
             test_poi, data, model, init_pars, par_bounds, fixed_params
         )
-    assert (
-        "No POI is defined. A POI is required for profile likelihood based test statistics."
-        in str(excinfo.value)
-    )
 
-    with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.UnspecifiedPOI,
+        match="No POI is defined. A POI is required for profile likelihood based test statistics.",
+    ):
         pyhf.infer.test_statistics.qmu(
             test_poi, data, model, init_pars, par_bounds, fixed_params
         )
-    assert (
-        "No POI is defined. A POI is required for profile likelihood based test statistics."
-        in str(excinfo.value)
-    )
 
-    with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.UnspecifiedPOI,
+        match="No POI is defined. A POI is required for profile likelihood based test statistics.",
+    ):
         pyhf.infer.test_statistics.qmu_tilde(
             test_poi, data, model, init_pars, par_bounds, fixed_params
         )
-    assert (
-        "No POI is defined. A POI is required for profile likelihood based test statistics."
-        in str(excinfo.value)
-    )
 
-    with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.UnspecifiedPOI,
+        match="No POI is defined. A POI is required for profile likelihood based test statistics.",
+    ):
         pyhf.infer.test_statistics.tmu(
             test_poi, data, model, init_pars, par_bounds, fixed_params
         )
-    assert (
-        "No POI is defined. A POI is required for profile likelihood based test statistics."
-        in str(excinfo.value)
-    )
 
-    with pytest.raises(pyhf.exceptions.UnspecifiedPOI) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.UnspecifiedPOI,
+        match="No POI is defined. A POI is required for profile likelihood based test statistics.",
+    ):
         pyhf.infer.test_statistics.tmu_tilde(
             test_poi, data, model, init_pars, par_bounds, fixed_params
         )
-    assert (
-        "No POI is defined. A POI is required for profile likelihood based test statistics."
-        in str(excinfo.value)
-    )
 
 
 @pytest.mark.parametrize("test_stat", ["qtilde", "q"])

tests/test_utils.py

@@ -37,15 +37,13 @@ def test_digest(obj, algorithm):
 
 
 def test_digest_bad_obj():
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(ValueError, match="not JSON-serializable"):
         pyhf.utils.digest(object())
-    assert 'not JSON-serializable' in str(excinfo.value)
 
 
 def test_digest_bad_alg():
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(ValueError, match="nonexistent_algorithm"):
         pyhf.utils.digest({}, algorithm='nonexistent_algorithm')
-    assert 'nonexistent_algorithm' in str(excinfo.value)
 
 
 @pytest.mark.parametrize('oneline', [False, True])

tests/test_workspace.py

@@ -79,24 +79,25 @@ def test_get_measurement(workspace_factory):
 
 def test_get_measurement_nonexist(workspace_factory):
     w = workspace_factory()
-    with pytest.raises(pyhf.exceptions.InvalidMeasurement) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.InvalidMeasurement, match="nonexistent_measurement"
+    ):
         w.get_measurement(measurement_name='nonexistent_measurement')
-    assert 'nonexistent_measurement' in str(excinfo.value)
 
 
 def test_get_measurement_index_outofbounds(workspace_factory):
     ws = workspace_factory()
-    with pytest.raises(pyhf.exceptions.InvalidMeasurement) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidMeasurement, match="out of bounds"):
         ws.get_measurement(measurement_index=9999)
-    assert 'out of bounds' in str(excinfo.value)
 
 
 def test_get_measurement_no_measurements_defined(workspace_factory):
     ws = workspace_factory()
     ws.measurement_names = []
-    with pytest.raises(pyhf.exceptions.InvalidMeasurement) as excinfo:
+    with pytest.raises(
+        pyhf.exceptions.InvalidMeasurement, match="No measurements have been defined"
+    ):
         ws.get_measurement()
-    assert 'No measurements have been defined' in str(excinfo.value)
 
 
 def test_get_workspace_measurement_priority(workspace_factory):
@@ -413,9 +414,8 @@ def test_combine_workspace_same_channels_incompatible_structure(
     new_ws = ws.rename(
         samples={ws.samples[0]: 'sample_other'},
     )
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation, match="channel1"):
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert 'channel1' in str(excinfo.value)
 
 
 @pytest.mark.parametrize("join", ['outer', 'left outer', 'right outer'])
@@ -444,9 +444,8 @@ def test_combine_workspace_incompatible_poi(workspace_factory, join):
         channels={channel: f'renamed_{channel}' for channel in ws.channels},
         modifiers={ws.get_measurement()['config']['poi']: 'renamedPOI'},
     )
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation, match="GaussExample"):
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert 'GaussExample' in str(excinfo.value)
 
 
 @pytest.mark.parametrize("join", ['none', 'outer', 'left outer', 'right outer'])
@@ -467,10 +466,12 @@ def test_combine_workspace_diff_version(workspace_factory, join):
         },
     )
     new_ws['version'] = '1.2.0'
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as exc_info:
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert '1.0.0' in str(excinfo.value)
-    assert '1.2.0' in str(excinfo.value)
+    # Using asserts with str(exc_info.value) over pytest.raises(..., match="...")
+    # to make it easier to check multiple strings.
+    assert "1.0.0" in str(exc_info.value)
+    assert "1.2.0" in str(exc_info.value)
 
 
 @pytest.mark.parametrize("join", ['none'])
@@ -479,9 +480,8 @@ def test_combine_workspace_duplicate_parameter_configs(workspace_factory, join):
     new_ws = ws.rename(
         channels={channel: f'renamed_{channel}' for channel in ws.channels},
     )
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation, match="GaussExample"):
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert 'GaussExample' in str(excinfo.value)
 
 
 @pytest.mark.parametrize("join", ['outer', 'left outer', 'right outer'])
@@ -570,9 +570,8 @@ def test_combine_workspace_invalid_join_operation(workspace_factory, join):
     new_ws = ws.rename(
         channels={channel: f'renamed_{channel}' for channel in ws.channels},
     )
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(ValueError, match=join):
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert join in str(excinfo.value)
 
 
 @pytest.mark.parametrize("join", ['none'])
@@ -581,9 +580,8 @@ def test_combine_workspace_invalid_join_operation_merge(workspace_factory, join)
     new_ws = ws.rename(
         channels={channel: f'renamed_{channel}' for channel in ws.channels},
     )
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(ValueError, match=join):
         pyhf.Workspace.combine(ws, new_ws, join=join, merge_channels=True)
-    assert join in str(excinfo.value)
 
 
 @pytest.mark.parametrize("join", ['none'])
@@ -595,9 +593,8 @@ def test_combine_workspace_incompatible_parameter_configs(workspace_factory, joi
     new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
         'bounds'
     ] = [[0.0, 1.0]]
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation, match="GaussExample"):
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert 'GaussExample' in str(excinfo.value)
 
 
 @pytest.mark.parametrize("join", ['outer'])
@@ -611,15 +608,17 @@ def test_combine_workspace_incompatible_parameter_configs_outer_join(
     new_ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][0][
         'bounds'
     ] = [[0.0, 1.0]]
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as exc_info:
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert 'GaussExample' in str(excinfo.value)
-    assert ws.get_measurement(measurement_name='GaussExample')['config']['parameters'][
+    # Using asserts with str(exc_info.value) over pytest.raises(..., match="...")
+    # to make it easier to check multiple strings.
+    assert "GaussExample" in str(exc_info.value)
+    assert ws.get_measurement(measurement_name="GaussExample")["config"]["parameters"][
         0
-    ]['name'] in str(excinfo.value)
-    assert new_ws.get_measurement(measurement_name='GaussExample')['config'][
-        'parameters'
-    ][0]['name'] in str(excinfo.value)
+    ]["name"] in str(exc_info.value)
+    assert new_ws.get_measurement(measurement_name="GaussExample")["config"][
+        "parameters"
+    ][0]["name"] in str(exc_info.value)
 
 
 @pytest.mark.parametrize("join", ['outer'])
@@ -712,10 +711,12 @@ def test_combine_workspace_incompatible_observations(workspace_factory, join):
     )
     new_ws['observations'][0]['name'] = ws['observations'][0]['name']
     new_ws['observations'][0]['data'][0] = -10.0
-    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as excinfo:
+    with pytest.raises(pyhf.exceptions.InvalidWorkspaceOperation) as exc_info:
         pyhf.Workspace.combine(ws, new_ws, join=join)
-    assert ws['observations'][0]['name'] in str(excinfo.value)
-    assert 'observations' in str(excinfo.value)
+    # Using asserts with str(exc_info.value) over pytest.raises(..., match="...")
+    # to make it easier to check multiple strings.
+    assert ws["observations"][0]["name"] in str(exc_info.value)
+    assert "observations" in str(exc_info.value)
 
 
 def test_combine_workspace_incompatible_observations_left_outer(workspace_factory):