Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Mark array_axis_vcs and verilog_ams examples as xfail #751

Merged
merged 7 commits into from
Oct 8, 2021
Merged

Mark array_axis_vcs and verilog_ams examples as xfail #751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
bb1c1eb
tests/acceptance/test_external_run_scripts: mark array_axis_vcs examp…
umarcor Oct 1, 2021
8f98d4c
tests/acceptance/text_external_run_scripts: use pytest.mark.skipif in…
umarcor Oct 1, 2021
bf5c6d5
tests/acceptance/test_external_run_scripts: update reason 'Verilog'
umarcor Oct 3, 2021
60a0920
docs: add intersphinx mapping 'pytest'
umarcor Oct 3, 2021
892e3e2
docs/contributing: clarify that pytest is required for running accept…
umarcor Oct 3, 2021
fc4d9ca
tests/acceptance/test_external_run_scripts: cleanup
umarcor Oct 3, 2021
183471c
tests/acceptance/test_external_run_scripts: reorganise
umarcor Oct 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@

intersphinx_mapping = {
"python": ("https://docs.python.org/3.8/", None),
"pytest": ("https://docs.pytest.org/en/latest/", None),
}

# -- ExtLinks -------------------------------------------------------------
Expand Down
5 changes: 4 additions & 1 deletion docs/contributing.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ The test suite is divided into three parts:

The test suites must pass with Python 3.x as well as with all supported simulators.

For running the test locally we recommend using `pytest <https://pypi.python.org/pypi/pytest>`__.
For running the tests locally we recommend using `pytest <https://pypi.python.org/pypi/pytest>`__.
In fact, ``pytest`` is required for running the acceptance tests, because some specific features are used for allowing
certain tests to fail.
See :doc:`pytest:how-to/skipping`.

Example
'''''''
Expand Down
148 changes: 77 additions & 71 deletions tests/acceptance/test_external_run_scripts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
Verify that all external run scripts work correctly
"""

import unittest
from unittest import TestCase
from pytest import mark
from pathlib import Path
from os import environ
from subprocess import call
Expand All @@ -29,22 +30,39 @@ def simulator_supports_verilog():


# pylint: disable=too-many-public-methods
@unittest.skipUnless(has_simulator(), "Requires simulator")
class TestExternalRunScripts(unittest.TestCase):
@mark.skipif(not has_simulator(), reason="Requires simulator")
class TestExternalRunScripts(TestCase):
"""
Verify that example projects run correctly
"""

def test_vhdl_uart_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "uart" / "run.py"))
@mark.skipif(not simulator_supports_verilog(), reason="Requires a Verilog simulator")
def test_verilog_user_guide_example_project(self):
self.check(ROOT / "examples/verilog/user_guide/run.py", exit_code=1)
check_report(
self.report_file,
[
("passed", "lib.tb_example_basic.all"),
("passed", "lib.tb_example.Test that a successful test case passes"),
(
"failed",
"lib.tb_example.Test that a failing test case actually fails",
),
(
"failed",
"lib.tb_example.Test that a test case that takes too long time fails with a timeout",
),
],
)

@unittest.skipUnless(simulator_supports_verilog(), "Verilog")
@mark.skipif(not simulator_supports_verilog(), reason="Requires a Verilog simulator")
def test_verilog_uart_example_project(self):
self.check(str(ROOT / "examples" / "verilog" / "uart" / "run.py"))
self.check(ROOT / "examples/verilog/uart/run.py")

@unittest.skipUnless(simulator_supports_verilog(), "Verilog")
@mark.skipif(not simulator_supports_verilog(), reason="Requires a Verilog simulator")
@mark.xfail(reason="Requires AMS")
def test_verilog_ams_example(self):
self.check(str(ROOT / "examples" / "verilog" / "verilog_ams" / "run.py"))
self.check(ROOT / "examples/verilog/verilog_ams/run.py")
check_report(
self.report_file,
[
Expand All @@ -53,11 +71,14 @@ def test_verilog_ams_example(self):
],
)

def test_vhdl_uart_example_project(self):
self.check(ROOT / "examples/vhdl/uart/run.py")

def test_vhdl_logging_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "logging" / "run.py"))
self.check(ROOT / "examples/vhdl/logging/run.py")

def test_vhdl_run_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "run" / "run.py"), exit_code=1)
self.check(ROOT / "examples/vhdl/run/run.py", exit_code=1)
check_report(
self.report_file,
[
Expand Down Expand Up @@ -100,7 +121,7 @@ def test_vhdl_run_example_project(self):

def test_vhdl_third_party_integration_example_project(self):
self.check(
str(ROOT / "examples" / "vhdl" / "third_party_integration" / "run.py"),
ROOT / "examples/vhdl/third_party_integration/run.py",
exit_code=1,
)
check_report(
Expand All @@ -119,17 +140,17 @@ def test_vhdl_third_party_integration_example_project(self):
)

def test_vhdl_check_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "check" / "run.py"))
self.check(ROOT / "examples/vhdl/check/run.py")

@unittest.skipIf(
@mark.skipif(
simulator_check(lambda simclass: not simclass.supports_coverage()),
"This simulator/backend does not support coverage",
reason="This simulator/backend does not support coverage",
)
def test_vhdl_coverage_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "coverage" / "run.py"))
self.check(ROOT / "examples/vhdl/coverage/run.py")

def test_vhdl_generate_tests_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "generate_tests" / "run.py"))
self.check(ROOT / "examples/vhdl/generate_tests/run.py")
check_report(
self.report_file,
[
Expand All @@ -146,7 +167,7 @@ def test_vhdl_generate_tests_example_project(self):
)

def test_vhdl_composite_generics_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "composite_generics" / "run.py"))
self.check(ROOT / "examples/vhdl/composite_generics/run.py")
check_report(
self.report_file,
[
Expand All @@ -155,25 +176,29 @@ def test_vhdl_composite_generics_example_project(self):
],
)

@unittest.skipUnless(simulator_is("ghdl"), "Support complex JSON strings as generic")
@mark.skipif(not simulator_is("ghdl"), reason="Support complex JSON strings as generic")
def test_vhdl_json4vhdl_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "json4vhdl" / "run.py"))
self.check(ROOT / "examples/vhdl/json4vhdl/run.py")

def test_vhdl_array_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "array" / "run.py"))
self.check(ROOT / "examples/vhdl/array/run.py")

@mark.xfail(
not simulator_is("ghdl"),
reason="Only simulators with PSL functionality",
)
def test_vhdl_array_axis_vcs_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "array_axis_vcs" / "run.py"))
self.check(ROOT / "examples/vhdl/array_axis_vcs/run.py")

def test_vhdl_axi_dma_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "axi_dma" / "run.py"))
self.check(ROOT / "examples/vhdl/axi_dma/run.py")

@unittest.skipIf(
@mark.skipif(
simulator_check(lambda simclass: not simclass.supports_vhdl_contexts()),
"This simulator/backend does not support VHDL contexts",
reason="This simulator/backend does not support VHDL contexts",
)
def test_vhdl_user_guide_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "user_guide" / "run.py"), exit_code=1)
self.check(ROOT / "examples/vhdl/user_guide/run.py", exit_code=1)
check_report(
self.report_file,
[
Expand All @@ -185,7 +210,7 @@ def test_vhdl_user_guide_example_project(self):

def test_vhdl_user_guide_93_example_project(self):
self.check(
str(ROOT / "examples" / "vhdl" / "user_guide" / "vhdl1993" / "run.py"),
ROOT / "examples/vhdl/user_guide/vhdl1993/run.py",
exit_code=1,
)
check_report(
Expand All @@ -197,99 +222,80 @@ def test_vhdl_user_guide_93_example_project(self):
],
)

@unittest.skipUnless(simulator_supports_verilog(), "Verilog")
def test_verilog_user_guide_example_project(self):
self.check(str(ROOT / "examples" / "verilog" / "user_guide" / "run.py"), exit_code=1)
check_report(
self.report_file,
[
("passed", "lib.tb_example_basic.all"),
("passed", "lib.tb_example.Test that a successful test case passes"),
(
"failed",
"lib.tb_example.Test that a failing test case actually fails",
),
(
"failed",
"lib.tb_example.Test that a test case that takes too long time fails with a timeout",
),
],
)

def test_vhdl_com_example_project(self):
self.check(str(ROOT / "examples" / "vhdl" / "com" / "run.py"))
self.check(ROOT / "examples/vhdl/com/run.py")

def test_array_vhdl_2008(self):
self.check(str(VHDL_PATH / "array" / "run.py"))
self.check(VHDL_PATH / "array/run.py")

def test_data_types_vhdl_2008(self):
self.check(str(VHDL_PATH / "data_types" / "run.py"))
self.check(VHDL_PATH / "data_types/run.py")

def test_data_types_vhdl_2002(self):
self.check(str(VHDL_PATH / "data_types" / "run.py"), vhdl_standard="2002")
self.check(VHDL_PATH / "data_types/run.py", vhdl_standard="2002")

def test_data_types_vhdl_93(self):
self.check(str(VHDL_PATH / "data_types" / "run.py"), vhdl_standard="93")
self.check(VHDL_PATH / "data_types/run.py", vhdl_standard="93")

def test_random_vhdl_2008(self):
self.check(str(VHDL_PATH / "random" / "run.py"))
self.check(VHDL_PATH / "random/run.py")

def test_check_vhdl_2008(self):
self.check(str(VHDL_PATH / "check" / "run.py"))
self.check(VHDL_PATH / "check/run.py")

def test_check_vhdl_2002(self):
self.check(str(VHDL_PATH / "check" / "run.py"), vhdl_standard="2002")
self.check(VHDL_PATH / "check/run.py", vhdl_standard="2002")

def test_check_vhdl_93(self):
self.check(str(VHDL_PATH / "check" / "run.py"), vhdl_standard="93")
self.check(VHDL_PATH / "check/run.py", vhdl_standard="93")

def test_logging_vhdl_2008(self):
self.check(str(VHDL_PATH / "logging" / "run.py"))
self.check(VHDL_PATH / "logging/run.py")

def test_logging_vhdl_2002(self):
self.check(str(VHDL_PATH / "logging" / "run.py"), vhdl_standard="2002")
self.check(VHDL_PATH / "logging/run.py", vhdl_standard="2002")

def test_logging_vhdl_93(self):
self.check(str(VHDL_PATH / "logging" / "run.py"), vhdl_standard="93")
self.check(VHDL_PATH / "logging/run.py", vhdl_standard="93")

def test_run_vhdl_2008(self):
self.check(str(VHDL_PATH / "run" / "run.py"))
self.check(VHDL_PATH / "run/run.py")

def test_run_vhdl_2002(self):
self.check(str(VHDL_PATH / "run" / "run.py"), vhdl_standard="2002")
self.check(VHDL_PATH / "run/run.py", vhdl_standard="2002")

def test_run_vhdl_93(self):
self.check(str(VHDL_PATH / "run" / "run.py"), vhdl_standard="93")
self.check(VHDL_PATH / "run/run.py", vhdl_standard="93")

def test_string_ops_vhdl_2008(self):
self.check(str(VHDL_PATH / "string_ops" / "run.py"))
self.check(VHDL_PATH / "string_ops/run.py")

def test_string_ops_vhdl_2002(self):
self.check(str(VHDL_PATH / "string_ops" / "run.py"), vhdl_standard="2002")
self.check(VHDL_PATH / "string_ops/run.py", vhdl_standard="2002")

def test_string_ops_vhdl_93(self):
self.check(str(VHDL_PATH / "string_ops" / "run.py"), vhdl_standard="93")
self.check(VHDL_PATH / "string_ops/run.py", vhdl_standard="93")

def test_dictionary_vhdl_2008(self):
self.check(str(VHDL_PATH / "dictionary" / "run.py"))
self.check(VHDL_PATH / "dictionary/run.py")

def test_dictionary_vhdl_2002(self):
self.check(str(VHDL_PATH / "dictionary" / "run.py"), vhdl_standard="2002")
self.check(VHDL_PATH / "dictionary/run.py", vhdl_standard="2002")

def test_dictionary_vhdl_93(self):
self.check(str(VHDL_PATH / "dictionary" / "run.py"), vhdl_standard="93")
self.check(VHDL_PATH / "dictionary/run.py", vhdl_standard="93")

def test_path_vhdl_2008(self):
self.check(str(VHDL_PATH / "path" / "run.py"))
self.check(VHDL_PATH / "path/run.py")

def test_path_vhdl_2002(self):
self.check(str(VHDL_PATH / "path" / "run.py"), vhdl_standard="2002")
self.check(VHDL_PATH / "path/run.py", vhdl_standard="2002")

def test_path_vhdl_93(self):
self.check(str(VHDL_PATH / "path" / "run.py"), vhdl_standard="93")
self.check(VHDL_PATH / "path/run.py", vhdl_standard="93")

def test_com_vhdl_2008(self):
self.check(str(VHDL_PATH / "com" / "run.py"))
self.check(VHDL_PATH / "com/run.py")

def setUp(self):
self.output_path = str(Path(__file__).parent / "external_run_out")
Expand Down