From 35ed4729757e6526b5066e4d11dd1dc9f45e3c90 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 18:07:52 +0100 Subject: [PATCH 01/10] first attempt at generating sparse CI run matrix in python; also, cancel previous runs on the same branch --- .github/generate-job-matrix.py | 168 ++++++++++++++++++++ .github/job_matrix.py | 113 +++++++++++++ .github/workflows/ci-clang-tidy.yml | 41 ++--- .github/workflows/ci-conan.yml | 158 +++--------------- .github/workflows/ci-freestanding.yml | 55 +++---- .github/workflows/ci-test-package-cmake.yml | 144 +++-------------- 6 files changed, 367 insertions(+), 312 deletions(-) create mode 100644 .github/generate-job-matrix.py create mode 100644 .github/job_matrix.py diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py new file mode 100644 index 0000000000..b2465d0565 --- /dev/null +++ b/.github/generate-job-matrix.py @@ -0,0 +1,168 @@ +import argparse +import json +import typing +import itertools +import random +import dataclasses +from types import SimpleNamespace +from dataclasses import dataclass + +from job_matrix import Configuration, Compiler, MatrixElement, CombinationCollector, dataclass_to_json + +def make_gcc_config(version: int) -> Configuration: + return Configuration( + name=f"GCC-{version}", + os="ubuntu-24.04", + compiler=Compiler( + type="GCC", + version=version, + cc=f"gcc-{version}", + cxx=f"g++-{version}", + ), + cxx_modules=False, + std_format_support=version >= 13, + ) + + +def make_clang_config(version: int, platform: typing.Literal["x86-64", "arm64"] = "x86-64") -> Configuration: + ret = SimpleNamespace( + name=f"Clang-{version} ({platform})", + os=None, # replaced below + compiler=SimpleNamespace( + type="CLANG", + version=version, + cc=f"clang-{version}", + cxx=f"clang++-{version}", + ), + lib="libc++", + cxx_modules=version >= 17, + std_format_support=version >= 17, + ) + match platform: + case "x86-64": + ret.os = "ubuntu-24.04" + case "arm64": + ret.os = "macos-14" + pfx = f"/opt/homebrew/opt/llvm@{version}/bin/" + ret.compiler.cc = pfx + ret.compiler.cc + ret.compiler.cxx = pfx + ret.compiler.cxx + case _: + raise KeyError(f"Unsupported platform {platform!r} for Clang") + ret.compiler = Compiler(**vars(ret.compiler)) + return Configuration(**vars(ret)) + + +def make_apple_clang_config(version: int) -> Configuration: + ret = Configuration( + name=f"Apple Clang {version}", + os="macos-13", + compiler=Compiler( + type="APPLE_CLANG", + version=f"{version}.0", + cc="clang", + cxx="clang++", + ), + cxx_modules=False, + std_format_support=False, + ) + return ret + + +def make_msvc_config(release: str, version: int) -> Configuration: + ret = Configuration( + name=f"MSVC {release}", + os="windows-2022", + compiler=Compiler( + type="MSVC", + version=version, + cc="", + cxx="", + ), + cxx_modules=False, + std_format_support=True, + ) + return ret + + +configs = {c.name: c for c in [make_gcc_config(ver) for ver in [12, 13, 14]] + + [make_clang_config(ver, platform) for ver in [16, 17, 18, 19] for platform in ["x86-64", "arm64"]] + + [make_apple_clang_config(ver) for ver in [15]] + + [make_msvc_config(release="14.4", version=194)]} + +full_matrix = dict( + config=list(configs.values()), + std=[20, 23], + formatting=["std::format", "fmtlib"], + contracts=["none", "gsl-lite", "ms-gsl"], + build_type=["Release", "Debug"], +) + + +def main(): + parser = argparse.ArgumentParser() + # parser.add_argument("-I","--include",nargs="+",action="append") + # parser.add_argument("-X","--exclude",nargs="+",action="append") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--preset", default=None) + parser.add_argument("--debug", nargs="+", default=["combinations"]) + parser.add_argument("--suppress-output", default=False, action="store_true") + + args = parser.parse_args() + + rgen = random.Random(args.seed) + + collector = CombinationCollector(full_matrix) + match args.preset: + case None: + pass + case "all": + collector.all_combinations() + case "conan" | "cmake": + collector.all_combinations(formatting="std::format", contracts="gsl-lite", build_type="Debug", std=20) + collector.all_combinations( + filter=lambda me: not me.config.std_format_support, + formatting="fmtlib", contracts="gsl-lite", build_type="Debug", std=20, + ) + collector.sample_combinations(rgen=rgen, min_samples_per_value=2) + case "clang-tidy": + collector.all_combinations(config=configs["Clang-18 (x86-64)"]) + case "freestanding": + collector.all_combinations( + config=[configs[c] for c in ["GCC-14", "Clang-18 (x86-64)"]], + contracts="none", + std=23, + ) + case _: + raise KeyError(f"Unsupported preset {args.preset!r}") + + if not collector.combinations: + raise ValueError(f"No combination has been produced") + + data = sorted(collector.combinations) + + if not args.suppress_output: + print(f"::set-output name=matrix::{json.dumps(data, default=dataclass_to_json)}") + + for dbg in args.debug: + match dbg: + case "yaml": + import yaml + json_data = json.loads(json.dumps(data, default=dataclass_to_json)) + print(yaml.safe_dump(json_data)) + case "json": + print(json.dumps(data, default=dataclass_to_json, indent=4)) + case "combinations": + for e in data: + print(f"{e.config!s:17s} c++{e.std:2d} {e.formatting:11s} {e.contracts:8s} {e.build_type:8s}") + case "counts": + print(f"Total combinations {len(data)}") + for (k, v), n in sorted(collector.per_value_counts.items()): + print(f" {k}={v}: {n}") + case "none": + pass + case _: + raise KeyError(f"Unknown debug mode {dbg!r}") + + +if __name__ == "__main__": + main() diff --git a/.github/job_matrix.py b/.github/job_matrix.py new file mode 100644 index 0000000000..9065f3892b --- /dev/null +++ b/.github/job_matrix.py @@ -0,0 +1,113 @@ +import argparse +import json +import typing +import itertools +import random +import dataclasses +from types import SimpleNamespace +from dataclasses import dataclass + + +@dataclass(frozen=True, order=True) +class Compiler: + type: typing.Literal["GCC", "CLANG", "APPLE_CLANG", "MSVC"] + version: str | int + cc: str + cxx: str + + +@dataclass(frozen=True, order=True) +class Configuration: + name: str + os: str + compiler: Compiler + cxx_modules: bool + std_format_support: bool + conan_config: str = "" + lib: typing.Literal["libc++", "libstdc++"] | None = None + + def __str__(self): + return self.name + +@dataclass(frozen=True, order=True) +class MatrixElement: + config: Configuration + std: typing.Literal[20, 23] + formatting: typing.Literal["std::format", "fmtlib"] + contracts: typing.Literal["none", "gsl-lite", "ms-gsl"] + build_type: typing.Literal["Release", "Debug"] + + +def dataclass_to_json(obj): + """ Convert dataclasses to something json-serialisable """ + if dataclasses.is_dataclass(obj): + return dataclasses.asdict(obj) + raise TypeError(f"Unknown object of type {type(obj).__name__}") + + +class CombinationCollector: + """ Incremental builder of MatrixElements, allowing successive selection of entries. + """ + + def __init__(self, full_matrix: dict[str, list[typing.Any]]): + self.full_matrix = full_matrix + self.combinations: set[MatrixElement] = set() + self.per_value_counts: dict[tuple[str, typing.Any], int] = {(k, v): 0 for k, options in full_matrix.items() for + v in options} + + def _make_submatrix(self, **overrides): + new_matrix = dict(self.full_matrix) + for k, v in overrides.items(): + if not isinstance(v, list): + v = [v] + new_matrix[k] = v + return new_matrix + + def _add_combination(self, e: MatrixElement): + if e in self.combinations or (e.formatting == "std::format" and not e.config.std_format_support): + return + self.combinations.add(e) + # update per_value_counts + for k, v in vars(e).items(): + idx = (k, v) + self.per_value_counts[idx] = self.per_value_counts.get(idx, 0) + 1 + + def all_combinations(self, *, filter: typing.Callable[[MatrixElement], bool] | None = None, **overrides): + """ Adds all combinations in the submatrix defined by `overrides`. """ + matrix = self._make_submatrix(**overrides) + keys = tuple(matrix.keys()) + for combination in itertools.product(*matrix.values()): + cand = MatrixElement(**dict(zip(keys, combination))) + if filter and not filter(cand): + continue + self._add_combination(cand) + + def sample_combinations(self, *, rgen: random.Random, min_samples_per_value: int = 1, + filter: typing.Callable[[MatrixElement], bool] | None = None, **overrides): + """ Adds samples from the submatrix defined by `overrides`, ensuring each individual value appears at least n times. """ + matrix = self._make_submatrix(**overrides) + missing: dict[tuple[str, typing.Any], int] = {} + for key, options in matrix.items(): + for value in options: + idx = (key, value) + missing[idx] = min_samples_per_value - self.per_value_counts.get(idx, 0) + while missing: + (force_key, force_option), remaining = next(iter(missing.items())) + if remaining <= 0: + missing.pop((force_key, force_option)) + continue + choice = {} + for key, options in matrix.items(): + choice[key] = force_option if key == force_key else rgen.choice(options) + cand = MatrixElement(**choice) + if filter and not filter(cand): + continue + self._add_combination(cand) + for idx in choice.items(): + if missing.pop(idx, 0) <= 0: + continue + remaining = min_samples_per_value - self.per_value_counts.get(idx, 0) + if remaining > 0: + missing[idx] = remaining + + diff --git a/.github/workflows/ci-clang-tidy.yml b/.github/workflows/ci-clang-tidy.yml index 015d42296a..ddf7f23fc4 100644 --- a/.github/workflows/ci-clang-tidy.yml +++ b/.github/workflows/ci-clang-tidy.yml @@ -35,32 +35,33 @@ on: - "docs/**" jobs: + cancel-previous: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" + generate-matrix: + name: "Generate build matrix for ${{ github.workflow }}" + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + runs-on: ubuntu-24.04 + needs: cancel-previous + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - id: set-matrix + run: python .github/generate-job-matrix.py --preset clang-tidy --seed 42 --debug combinations counts build: name: "${{ matrix.formatting }} ${{ matrix.contracts }} C++${{ matrix.std }} ${{ matrix.config.name }} ${{ matrix.build_type }}" runs-on: ${{ matrix.config.os }} + needs: generate-matrix strategy: fail-fast: false matrix: - formatting: ["std::format", "fmtlib"] - contracts: ["none", "gsl-lite", "ms-gsl"] - std: [20, 23] - config: - - { - name: "Clang-18", - os: ubuntu-24.04, - compiler: - { - type: CLANG, - version: 18, - cc: "clang-18", - cxx: "clang++-18", - }, - lib: "libc++", - cxx_modules: "False", - std_format_support: "True", - conan-config: "", - } - build_type: ["Release", "Debug"] + include: ${{fromJson(needs.generate-matrix.outputs.matrix)}} env: CC: ${{ matrix.config.compiler.cc }} diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml index 8d5fc75a46..f5377fc276 100644 --- a/.github/workflows/ci-conan.yml +++ b/.github/workflows/ci-conan.yml @@ -34,148 +34,36 @@ env: CHANNEL: ${{ fromJSON('["testing", "stable"]')[github.ref_type == 'tag' && startsWith(github.ref_name, 'v')] }} jobs: + cancel-previous: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" + generate-matrix: + name: "Generate build matrix for ${{ github.workflow }}" + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + runs-on: ubuntu-24.04 + needs: cancel-previous + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - id: set-matrix + run: python .github/generate-job-matrix.py --preset conan --seed 42 --debug combinations counts build: name: "${{ matrix.formatting }} ${{ matrix.contracts }} C++${{ matrix.std }} ${{ matrix.config.name }} ${{ matrix.build_type }}" runs-on: ${{ matrix.config.os }} + needs: generate-matrix strategy: fail-fast: false matrix: - formatting: ["std::format", "fmtlib"] - contracts: ["none", "gsl-lite", "ms-gsl"] - std: [20, 23] - config: - - { - name: "MSVC 14.4", - os: windows-2022, - compiler: { type: MSVC, version: 194, cc: "", cxx: "" }, - cxx_modules: "False", - std_format_support: "True", - conan-config: "", - } - - { - name: "GCC-12", - os: ubuntu-24.04, - compiler: - { - type: GCC, - version: 12, - cc: "gcc-12", - cxx: "g++-12", - }, - cxx_modules: "False", - std_format_support: "False", - conan-config: "", - } - - { - name: "GCC-13", - os: ubuntu-24.04, - compiler: - { - type: GCC, - version: 13, - cc: "gcc-13", - cxx: "g++-13", - }, - cxx_modules: "False", - std_format_support: "True", - conan-config: "", - } - - { - name: "GCC-14", - os: ubuntu-24.04, - compiler: - { - type: GCC, - version: 14, - cc: "gcc-14", - cxx: "g++-14", - }, - cxx_modules: "False", - std_format_support: "True", - conan-config: "", - } - - { - name: "Clang-16", - os: ubuntu-22.04, - compiler: - { - type: CLANG, - version: 16, - cc: "clang-16", - cxx: "clang++-16", - }, - lib: "libc++", - cxx_modules: "False", - std_format_support: "False", - conan-config: "", - } - - { - name: "Clang-17", - os: ubuntu-24.04, - compiler: - { - type: CLANG, - version: 17, - cc: "clang-17", - cxx: "clang++-17", - }, - lib: "libc++", - cxx_modules: "True", - std_format_support: "True", - conan-config: "", - } - - { - name: "Clang-18", - os: ubuntu-24.04, - compiler: - { - type: CLANG, - version: 18, - cc: "clang-18", - cxx: "clang++-18", - }, - lib: "libc++", - cxx_modules: "True", - std_format_support: "True", - conan-config: "", - } - - { - name: "Clang-18 on Apple M1 (arm64)", - os: macos-14, - compiler: - { - type: CLANG, - version: 18, - cc: "/opt/homebrew/opt/llvm@18/bin/clang-18", - cxx: "/opt/homebrew/opt/llvm@18/bin/clang++", - }, - lib: "libc++", - cxx_modules: "False", - std_format_support: "True" - } - - { - name: "Apple Clang 15", - os: macos-13, - compiler: - { - type: APPLE_CLANG, - version: "15.0", - cc: "clang", - cxx: "clang++", - }, - cxx_modules: "False", - std_format_support: "False", - conan-config: "", - } - build_type: ["Release", "Debug"] - exclude: - - formatting: "std::format" - config: { std_format_support: "False" } - + include: ${{fromJson(needs.generate-matrix.outputs.matrix)}} env: CC: ${{ matrix.config.compiler.cc }} CXX: ${{ matrix.config.compiler.cxx }} - steps: - uses: actions/checkout@v4 - name: Generate unique cache id @@ -265,14 +153,14 @@ jobs: run: | conan create . --user mpusz --channel ${CHANNEL} --lockfile-out=package.lock \ -b mp-units/* -b missing -c tools.cmake.cmaketoolchain:generator="Ninja Multi-Config" -c user.mp-units.build:all=True \ - -o '&:cxx_modules=${{ matrix.config.cxx_modules }}' -o '&:import_std=${{ env.import_std }}' -o '&:std_format=${{ env.std_format }}' -o '&:contracts=${{ matrix.contracts }}' ${{ matrix.config.conan-config }} + -o '&:cxx_modules=${{ matrix.config.cxx_modules }}' -o '&:import_std=${{ env.import_std }}' -o '&:std_format=${{ env.std_format }}' -o '&:contracts=${{ matrix.contracts }}' ${{ matrix.config.conan_config }} - name: Create Conan package if: matrix.config.compiler.type == 'MSVC' shell: bash run: | conan create . --user mpusz --channel ${CHANNEL} --lockfile-out=package.lock \ -b mp-units/* -b missing -c tools.cmake.cmaketoolchain:generator="Ninja Multi-Config" -c user.mp-units.build:all=False \ - -o '&:cxx_modules=${{ matrix.config.cxx_modules }}' -o '&:import_std=${{ env.import_std }}' -o '&:std_format=${{ env.std_format }}' -o '&:contracts=${{ matrix.contracts }}' ${{ matrix.config.conan-config }} + -o '&:cxx_modules=${{ matrix.config.cxx_modules }}' -o '&:import_std=${{ env.import_std }}' -o '&:std_format=${{ env.std_format }}' -o '&:contracts=${{ matrix.contracts }}' ${{ matrix.config.conan_config }} - name: Obtain package reference id: get-package-ref shell: bash diff --git a/.github/workflows/ci-freestanding.yml b/.github/workflows/ci-freestanding.yml index cde058db6d..5ce2662cbd 100644 --- a/.github/workflows/ci-freestanding.yml +++ b/.github/workflows/ci-freestanding.yml @@ -35,46 +35,33 @@ on: - "docs/**" jobs: + cancel-previous: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" + generate-matrix: + name: "Generate build matrix for ${{ github.workflow }}" + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + runs-on: ubuntu-24.04 + needs: cancel-previous + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - id: set-matrix + run: python .github/generate-job-matrix.py --preset freestanding --seed 42 --debug combinations counts build: name: "${{ matrix.formatting }} ${{ matrix.contracts }} C++${{ matrix.std }} ${{ matrix.config.name }} ${{ matrix.build_type }}" runs-on: ${{ matrix.config.os }} + needs: generate-matrix strategy: fail-fast: false matrix: - formatting: ["std::format", "fmtlib"] - contracts: ["none"] - std: [23] - config: - - { - name: "GCC-14", - os: ubuntu-24.04, - compiler: - { - type: GCC, - version: 14, - cc: "gcc-14", - cxx: "g++-14", - }, - cxx_modules: "False", - std_format_support: "True", - conan-config: "", - } - - { - name: "Clang-18", - os: ubuntu-24.04, - compiler: - { - type: CLANG, - version: 18, - cc: "clang-18", - cxx: "clang++-18", - }, - lib: "libc++", - cxx_modules: "True", - std_format_support: "True", - conan-config: "", - } - build_type: ["Release", "Debug"] + include: ${{fromJson(needs.generate-matrix.outputs.matrix)}} # TODO For some reason Clang-18 Debug with -ffreestanding does not pass CMakeTestCXXCompiler exclude: - build_type: "Debug" diff --git a/.github/workflows/ci-test-package-cmake.yml b/.github/workflows/ci-test-package-cmake.yml index d5bf5df0af..1644c495ac 100644 --- a/.github/workflows/ci-test-package-cmake.yml +++ b/.github/workflows/ci-test-package-cmake.yml @@ -39,135 +39,33 @@ on: - "test/**" jobs: + cancel-previous: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" + generate-matrix: + name: "Generate build matrix for ${{ github.workflow }}" + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + runs-on: ubuntu-24.04 + needs: cancel-previous + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - id: set-matrix + run: python .github/generate-job-matrix.py --preset conan --seed 42 --debug combinations counts test_package: name: "${{ matrix.formatting }} ${{ matrix.contracts }} C++${{ matrix.std }} ${{ matrix.config.name }} ${{ matrix.build_type }}" runs-on: ${{ matrix.config.os }} + needs: generate-matrix strategy: fail-fast: false matrix: - formatting: ["std::format", "fmtlib"] - contracts: ["none", "gsl-lite", "ms-gsl"] - std: [20, 23] - config: - - { - name: "MSVC 14.4", - os: windows-2022, - compiler: { type: MSVC, version: 194, cc: "", cxx: "" }, - cxx_modules: "False", - std_format_support: "True", - } - - { - name: "GCC-12", - os: ubuntu-24.04, - compiler: - { - type: GCC, - version: 12, - cc: "gcc-12", - cxx: "g++-12", - }, - cxx_modules: "False", - std_format_support: "False", - } - - { - name: "GCC-13", - os: ubuntu-24.04, - compiler: - { - type: GCC, - version: 13, - cc: "gcc-13", - cxx: "g++-13", - }, - cxx_modules: "False", - std_format_support: "True", - } - - { - name: "GCC-14", - os: ubuntu-24.04, - compiler: - { - type: GCC, - version: 14, - cc: "gcc-14", - cxx: "g++-14", - }, - cxx_modules: "False", - std_format_support: "True" - } - - { - name: "Clang-16", - os: ubuntu-22.04, - compiler: - { - type: CLANG, - version: 16, - cc: "clang-16", - cxx: "clang++-16", - }, - lib: "libc++", - cxx_modules: "False", - std_format_support: "False", - } - - { - name: "Clang-17", - os: ubuntu-24.04, - compiler: - { - type: CLANG, - version: 17, - cc: "clang-17", - cxx: "clang++-17", - }, - lib: "libc++", - cxx_modules: "False", - std_format_support: "True", - } - - { - name: "Clang-18", - os: ubuntu-24.04, - compiler: - { - type: CLANG, - version: 18, - cc: "clang-18", - cxx: "clang++-18", - }, - lib: "libc++", - cxx_modules: "False", - std_format_support: "True" - } - - { - name: "Clang-18 on Apple M1 (arm64)", - os: macos-14, - compiler: - { - type: CLANG, - version: 18, - cc: "/opt/homebrew/opt/llvm@18/bin/clang-18", - cxx: "/opt/homebrew/opt/llvm@18/bin/clang++", - }, - lib: "libc++", - cxx_modules: "False", - std_format_support: "True" - } - - { - name: "Apple Clang 15", - os: macos-14, - compiler: - { - type: APPLE_CLANG, - version: "15.0", - cc: "clang", - cxx: "clang++", - }, - cxx_modules: "False", - std_format_support: "False", - } - build_type: ["Release", "Debug"] - exclude: - - formatting: "std::format" - config: { std_format_support: "False" } + include: ${{fromJson(needs.generate-matrix.outputs.matrix)}} env: CC: ${{ matrix.config.compiler.cc }} From 7fa15d224d272462d2228f989a25cf813cff4bc9 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 18:43:50 +0100 Subject: [PATCH 02/10] fix formatting --- .flake8 | 3 ++ .github/generate-job-matrix.py | 50 ++++++++++++++------- .github/job_matrix.py | 47 +++++++++++-------- .github/workflows/ci-clang-tidy.yml | 1 + .github/workflows/ci-conan.yml | 1 + .github/workflows/ci-freestanding.yml | 1 + .github/workflows/ci-test-package-cmake.yml | 1 + 7 files changed, 71 insertions(+), 33 deletions(-) diff --git a/.flake8 b/.flake8 index 4592939f20..1404045659 100644 --- a/.flake8 +++ b/.flake8 @@ -7,3 +7,6 @@ ignore = E712, # line break before binary operator W503 +per-file-ignores = + # flake8 is just plain wrong here, contradicting black + .github/generate-job-matrix.py:E225,E231 diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py index b2465d0565..9ba3b8f3fe 100644 --- a/.github/generate-job-matrix.py +++ b/.github/generate-job-matrix.py @@ -1,13 +1,11 @@ import argparse import json -import typing -import itertools import random -import dataclasses +import typing from types import SimpleNamespace -from dataclasses import dataclass -from job_matrix import Configuration, Compiler, MatrixElement, CombinationCollector, dataclass_to_json +from job_matrix import CombinationCollector, Compiler, Configuration, dataclass_to_json + def make_gcc_config(version: int) -> Configuration: return Configuration( @@ -24,7 +22,9 @@ def make_gcc_config(version: int) -> Configuration: ) -def make_clang_config(version: int, platform: typing.Literal["x86-64", "arm64"] = "x86-64") -> Configuration: +def make_clang_config( + version: int, platform: typing.Literal["x86-64", "arm64"] = "x86-64" +) -> Configuration: ret = SimpleNamespace( name=f"Clang-{version} ({platform})", os=None, # replaced below @@ -84,10 +84,17 @@ def make_msvc_config(release: str, version: int) -> Configuration: return ret -configs = {c.name: c for c in [make_gcc_config(ver) for ver in [12, 13, 14]] - + [make_clang_config(ver, platform) for ver in [16, 17, 18, 19] for platform in ["x86-64", "arm64"]] - + [make_apple_clang_config(ver) for ver in [15]] - + [make_msvc_config(release="14.4", version=194)]} +configs = { + c.name: c + for c in [make_gcc_config(ver) for ver in [12, 13, 14]] + + [ + make_clang_config(ver, platform) + for ver in [16, 17, 18, 19] + for platform in ["x86-64", "arm64"] + ] + + [make_apple_clang_config(ver) for ver in [15]] + + [make_msvc_config(release="14.4", version=194)] +} full_matrix = dict( config=list(configs.values()), @@ -118,10 +125,18 @@ def main(): case "all": collector.all_combinations() case "conan" | "cmake": - collector.all_combinations(formatting="std::format", contracts="gsl-lite", build_type="Debug", std=20) + collector.all_combinations( + formatting="std::format", + contracts="gsl-lite", + build_type="Debug", + std=20, + ) collector.all_combinations( filter=lambda me: not me.config.std_format_support, - formatting="fmtlib", contracts="gsl-lite", build_type="Debug", std=20, + formatting="fmtlib", + contracts="gsl-lite", + build_type="Debug", + std=20, ) collector.sample_combinations(rgen=rgen, min_samples_per_value=2) case "clang-tidy": @@ -136,24 +151,29 @@ def main(): raise KeyError(f"Unsupported preset {args.preset!r}") if not collector.combinations: - raise ValueError(f"No combination has been produced") + raise ValueError("No combination has been produced") data = sorted(collector.combinations) if not args.suppress_output: - print(f"::set-output name=matrix::{json.dumps(data, default=dataclass_to_json)}") + print( + f"::set-output name=matrix::{json.dumps(data, default=dataclass_to_json)}" + ) for dbg in args.debug: match dbg: case "yaml": import yaml + json_data = json.loads(json.dumps(data, default=dataclass_to_json)) print(yaml.safe_dump(json_data)) case "json": print(json.dumps(data, default=dataclass_to_json, indent=4)) case "combinations": for e in data: - print(f"{e.config!s:17s} c++{e.std:2d} {e.formatting:11s} {e.contracts:8s} {e.build_type:8s}") + print( + f"{e.config!s:17s} c++{e.std:2d} {e.formatting:11s} {e.contracts:8s} {e.build_type:8s}" + ) case "counts": print(f"Total combinations {len(data)}") for (k, v), n in sorted(collector.per_value_counts.items()): diff --git a/.github/job_matrix.py b/.github/job_matrix.py index 9065f3892b..eb9562687c 100644 --- a/.github/job_matrix.py +++ b/.github/job_matrix.py @@ -1,10 +1,7 @@ -import argparse -import json -import typing +import dataclasses import itertools import random -import dataclasses -from types import SimpleNamespace +import typing from dataclasses import dataclass @@ -29,6 +26,7 @@ class Configuration: def __str__(self): return self.name + @dataclass(frozen=True, order=True) class MatrixElement: config: Configuration @@ -39,21 +37,21 @@ class MatrixElement: def dataclass_to_json(obj): - """ Convert dataclasses to something json-serialisable """ + """Convert dataclasses to something json-serialisable""" if dataclasses.is_dataclass(obj): return dataclasses.asdict(obj) raise TypeError(f"Unknown object of type {type(obj).__name__}") class CombinationCollector: - """ Incremental builder of MatrixElements, allowing successive selection of entries. - """ + """Incremental builder of MatrixElements, allowing successive selection of entries.""" def __init__(self, full_matrix: dict[str, list[typing.Any]]): self.full_matrix = full_matrix self.combinations: set[MatrixElement] = set() - self.per_value_counts: dict[tuple[str, typing.Any], int] = {(k, v): 0 for k, options in full_matrix.items() for - v in options} + self.per_value_counts: dict[tuple[str, typing.Any], int] = { + (k, v): 0 for k, options in full_matrix.items() for v in options + } def _make_submatrix(self, **overrides): new_matrix = dict(self.full_matrix) @@ -64,7 +62,9 @@ def _make_submatrix(self, **overrides): return new_matrix def _add_combination(self, e: MatrixElement): - if e in self.combinations or (e.formatting == "std::format" and not e.config.std_format_support): + if e in self.combinations or ( + e.formatting == "std::format" and not e.config.std_format_support + ): return self.combinations.add(e) # update per_value_counts @@ -72,8 +72,13 @@ def _add_combination(self, e: MatrixElement): idx = (k, v) self.per_value_counts[idx] = self.per_value_counts.get(idx, 0) + 1 - def all_combinations(self, *, filter: typing.Callable[[MatrixElement], bool] | None = None, **overrides): - """ Adds all combinations in the submatrix defined by `overrides`. """ + def all_combinations( + self, + *, + filter: typing.Callable[[MatrixElement], bool] | None = None, + **overrides, + ): + """Adds all combinations in the submatrix defined by `overrides`.""" matrix = self._make_submatrix(**overrides) keys = tuple(matrix.keys()) for combination in itertools.product(*matrix.values()): @@ -82,9 +87,17 @@ def all_combinations(self, *, filter: typing.Callable[[MatrixElement], bool] | N continue self._add_combination(cand) - def sample_combinations(self, *, rgen: random.Random, min_samples_per_value: int = 1, - filter: typing.Callable[[MatrixElement], bool] | None = None, **overrides): - """ Adds samples from the submatrix defined by `overrides`, ensuring each individual value appears at least n times. """ + def sample_combinations( + self, + *, + rgen: random.Random, + min_samples_per_value: int = 1, + filter: typing.Callable[[MatrixElement], bool] | None = None, + **overrides, + ): + """Adds samples from the submatrix defined by `overrides`, + ensuring each individual value appears at least n times. + """ matrix = self._make_submatrix(**overrides) missing: dict[tuple[str, typing.Any], int] = {} for key, options in matrix.items(): @@ -109,5 +122,3 @@ def sample_combinations(self, *, rgen: random.Random, min_samples_per_value: int remaining = min_samples_per_value - self.per_value_counts.get(idx, 0) if remaining > 0: missing[idx] = remaining - - diff --git a/.github/workflows/ci-clang-tidy.yml b/.github/workflows/ci-clang-tidy.yml index ddf7f23fc4..402455a1bd 100644 --- a/.github/workflows/ci-clang-tidy.yml +++ b/.github/workflows/ci-clang-tidy.yml @@ -39,6 +39,7 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + runs-on: ubuntu-24.04 steps: - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml index f5377fc276..f80f7776e3 100644 --- a/.github/workflows/ci-conan.yml +++ b/.github/workflows/ci-conan.yml @@ -38,6 +38,7 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + runs-on: ubuntu-24.04 steps: - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: diff --git a/.github/workflows/ci-freestanding.yml b/.github/workflows/ci-freestanding.yml index 5ce2662cbd..2b2f97da09 100644 --- a/.github/workflows/ci-freestanding.yml +++ b/.github/workflows/ci-freestanding.yml @@ -39,6 +39,7 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + runs-on: ubuntu-24.04 steps: - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: diff --git a/.github/workflows/ci-test-package-cmake.yml b/.github/workflows/ci-test-package-cmake.yml index 1644c495ac..bb1e960914 100644 --- a/.github/workflows/ci-test-package-cmake.yml +++ b/.github/workflows/ci-test-package-cmake.yml @@ -43,6 +43,7 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + runs-on: ubuntu-24.04 steps: - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: From e464677200cf8df116d05335370096696c5f6c28 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 18:58:06 +0100 Subject: [PATCH 03/10] don't test Clang 19 just yet; fix cancel-in-progres --- .github/generate-job-matrix.py | 2 +- .github/workflows/ci-clang-tidy.yml | 13 +++++-------- .github/workflows/ci-conan.yml | 12 ++++-------- .github/workflows/ci-freestanding.yml | 12 ++++-------- .github/workflows/ci-test-package-cmake.yml | 12 ++++-------- 5 files changed, 18 insertions(+), 33 deletions(-) diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py index 9ba3b8f3fe..ae06b9ce93 100644 --- a/.github/generate-job-matrix.py +++ b/.github/generate-job-matrix.py @@ -89,7 +89,7 @@ def make_msvc_config(release: str, version: int) -> Configuration: for c in [make_gcc_config(ver) for ver in [12, 13, 14]] + [ make_clang_config(ver, platform) - for ver in [16, 17, 18, 19] + for ver in [16, 17, 18] for platform in ["x86-64", "arm64"] ] + [make_apple_clang_config(ver) for ver in [15]] diff --git a/.github/workflows/ci-clang-tidy.yml b/.github/workflows/ci-clang-tidy.yml index 402455a1bd..849442b00d 100644 --- a/.github/workflows/ci-clang-tidy.yml +++ b/.github/workflows/ci-clang-tidy.yml @@ -34,20 +34,17 @@ on: paths-ignore: - "docs/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - cancel-previous: - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - runs-on: ubuntu-24.04 - steps: - - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: name: "Generate build matrix for ${{ github.workflow }}" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 - needs: cancel-previous steps: - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml index f80f7776e3..b439fa280d 100644 --- a/.github/workflows/ci-conan.yml +++ b/.github/workflows/ci-conan.yml @@ -33,20 +33,16 @@ on: env: CHANNEL: ${{ fromJSON('["testing", "stable"]')[github.ref_type == 'tag' && startsWith(github.ref_name, 'v')] }} +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - cancel-previous: - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - runs-on: ubuntu-24.04 - steps: - - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: name: "Generate build matrix for ${{ github.workflow }}" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 - needs: cancel-previous steps: - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/ci-freestanding.yml b/.github/workflows/ci-freestanding.yml index 2b2f97da09..f97d7cfb1f 100644 --- a/.github/workflows/ci-freestanding.yml +++ b/.github/workflows/ci-freestanding.yml @@ -34,20 +34,16 @@ on: paths-ignore: - "docs/**" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - cancel-previous: - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - runs-on: ubuntu-24.04 - steps: - - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: name: "Generate build matrix for ${{ github.workflow }}" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 - needs: cancel-previous steps: - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/ci-test-package-cmake.yml b/.github/workflows/ci-test-package-cmake.yml index bb1e960914..396ca2a4e6 100644 --- a/.github/workflows/ci-test-package-cmake.yml +++ b/.github/workflows/ci-test-package-cmake.yml @@ -38,20 +38,16 @@ on: - "example/**" - "test/**" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - cancel-previous: - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - runs-on: ubuntu-24.04 - steps: - - run: echo "Cancelling all previous runs of ${{ github.workflow }}-${{ github.ref }}" generate-matrix: name: "Generate build matrix for ${{ github.workflow }}" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 - needs: cancel-previous steps: - name: Set up Python uses: actions/setup-python@v5 From cc9ea9dd1a1dbf44372cb2623d24332777d5a751 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 19:05:04 +0100 Subject: [PATCH 04/10] add cancel-in-progress to all workflows --- .github/workflows/ci-formatting.yml | 4 ++++ .github/workflows/citation.yml | 4 ++++ .github/workflows/codeql.yml | 4 ++++ .github/workflows/documentation.yml | 5 +++++ 4 files changed, 17 insertions(+) diff --git a/.github/workflows/ci-formatting.yml b/.github/workflows/ci-formatting.yml index 086fe625a2..e61d5b8344 100644 --- a/.github/workflows/ci-formatting.yml +++ b/.github/workflows/ci-formatting.yml @@ -24,6 +24,10 @@ name: Formatting CI on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: check: runs-on: ubuntu-24.04 diff --git a/.github/workflows/citation.yml b/.github/workflows/citation.yml index e5f22acf74..de8eb51c28 100644 --- a/.github/workflows/citation.yml +++ b/.github/workflows/citation.yml @@ -5,6 +5,10 @@ on: paths: - CITATION.cff +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: Validate-CITATION-cff: runs-on: ubuntu-latest diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c7e0abb58e..e742e8a174 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -22,6 +22,10 @@ on: paths-ignore: - "docs/**" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: analyze: name: Analyze diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3072cd7967..09750e9ddd 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -36,6 +36,11 @@ on: - "mkdocs.yml" permissions: contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: deploy: runs-on: ubuntu-latest From a51462cc655a1be9ca960c36392aa18ebad97c45 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 19:09:12 +0100 Subject: [PATCH 05/10] missing checkout in generate-matrix step --- .github/workflows/ci-clang-tidy.yml | 1 + .github/workflows/ci-conan.yml | 1 + .github/workflows/ci-freestanding.yml | 1 + .github/workflows/ci-test-package-cmake.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/ci-clang-tidy.yml b/.github/workflows/ci-clang-tidy.yml index 849442b00d..589a123a92 100644 --- a/.github/workflows/ci-clang-tidy.yml +++ b/.github/workflows/ci-clang-tidy.yml @@ -46,6 +46,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 steps: + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml index b439fa280d..c5e0ad7ff2 100644 --- a/.github/workflows/ci-conan.yml +++ b/.github/workflows/ci-conan.yml @@ -44,6 +44,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 steps: + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/ci-freestanding.yml b/.github/workflows/ci-freestanding.yml index f97d7cfb1f..4fb016dc9b 100644 --- a/.github/workflows/ci-freestanding.yml +++ b/.github/workflows/ci-freestanding.yml @@ -45,6 +45,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 steps: + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/ci-test-package-cmake.yml b/.github/workflows/ci-test-package-cmake.yml index 396ca2a4e6..10e4e2edc6 100644 --- a/.github/workflows/ci-test-package-cmake.yml +++ b/.github/workflows/ci-test-package-cmake.yml @@ -49,6 +49,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} runs-on: ubuntu-24.04 steps: + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: From f4c8e901ac418a6b5d7582d9b9149e6d19026946 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 21:18:36 +0100 Subject: [PATCH 06/10] fix boolean conan options in dynamic CI matrix --- .github/generate-job-matrix.py | 12 ++++++------ .github/job_matrix.py | 21 +++++++++++++++------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py index ae06b9ce93..d711d3cce9 100644 --- a/.github/generate-job-matrix.py +++ b/.github/generate-job-matrix.py @@ -4,7 +4,7 @@ import typing from types import SimpleNamespace -from job_matrix import CombinationCollector, Compiler, Configuration, dataclass_to_json +from job_matrix import CombinationCollector, Compiler, Configuration def make_gcc_config(version: int) -> Configuration: @@ -155,20 +155,20 @@ def main(): data = sorted(collector.combinations) + json_data = [e.as_json() for e in data] + if not args.suppress_output: - print( - f"::set-output name=matrix::{json.dumps(data, default=dataclass_to_json)}" - ) + print(f"::set-output name=matrix::{json.dumps(json_data)}") for dbg in args.debug: match dbg: case "yaml": import yaml - json_data = json.loads(json.dumps(data, default=dataclass_to_json)) + json_data = json.loads(json.dumps(json_data)) print(yaml.safe_dump(json_data)) case "json": - print(json.dumps(data, default=dataclass_to_json, indent=4)) + print(json.dumps(json_data, indent=4)) case "combinations": for e in data: print( diff --git a/.github/job_matrix.py b/.github/job_matrix.py index eb9562687c..204be67999 100644 --- a/.github/job_matrix.py +++ b/.github/job_matrix.py @@ -35,12 +35,21 @@ class MatrixElement: contracts: typing.Literal["none", "gsl-lite", "ms-gsl"] build_type: typing.Literal["Release", "Debug"] - -def dataclass_to_json(obj): - """Convert dataclasses to something json-serialisable""" - if dataclasses.is_dataclass(obj): - return dataclasses.asdict(obj) - raise TypeError(f"Unknown object of type {type(obj).__name__}") + def as_json(self): + def dataclass_to_json(obj): + """Convert dataclasses to something json-serialisable""" + if dataclasses.is_dataclass(obj): + return { + k: dataclass_to_json(v) for k, v in dataclasses.asdict(obj).items() + } + return obj + + ret = dataclass_to_json(self) + # patch boolean conan configuration options + config = ret["config"] + for k in ["cxx_modules"]: + config[k] = "True" if config[k] else "False" + return ret class CombinationCollector: From 01f44c66fad892180fa9a706ab2506794afd7402 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 21:29:42 +0100 Subject: [PATCH 07/10] heed github warning, and use output file instead of set-output command; also, fix freestanding --- .github/generate-job-matrix.py | 9 ++++++++- .github/workflows/ci-freestanding.yml | 4 ---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py index d711d3cce9..ec819edeb5 100644 --- a/.github/generate-job-matrix.py +++ b/.github/generate-job-matrix.py @@ -1,5 +1,6 @@ import argparse import json +import os import random import typing from types import SimpleNamespace @@ -157,8 +158,14 @@ def main(): json_data = [e.as_json() for e in data] + output_file = os.environ.get("GITHUB_OUTPUT") if not args.suppress_output: - print(f"::set-output name=matrix::{json.dumps(json_data)}") + if output_file: + print(f"Writing outputs to {output_file}") + with open(output_file, "wt") as fh: + fh.write(f"matrix={json.dumps(json_data)}") + else: + print("No output file received!") for dbg in args.debug: match dbg: diff --git a/.github/workflows/ci-freestanding.yml b/.github/workflows/ci-freestanding.yml index 4fb016dc9b..7a234e023d 100644 --- a/.github/workflows/ci-freestanding.yml +++ b/.github/workflows/ci-freestanding.yml @@ -60,10 +60,6 @@ jobs: fail-fast: false matrix: include: ${{fromJson(needs.generate-matrix.outputs.matrix)}} - # TODO For some reason Clang-18 Debug with -ffreestanding does not pass CMakeTestCXXCompiler - exclude: - - build_type: "Debug" - config: { name: "Clang-18" } env: CC: ${{ matrix.config.compiler.cc }} From 5713243d075be0add1dec594c19caeaa48fac4f4 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 21:41:09 +0100 Subject: [PATCH 08/10] fix clang 16 --- .github/generate-job-matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py index ec819edeb5..b30ea573c7 100644 --- a/.github/generate-job-matrix.py +++ b/.github/generate-job-matrix.py @@ -41,7 +41,7 @@ def make_clang_config( ) match platform: case "x86-64": - ret.os = "ubuntu-24.04" + ret.os = "ubuntu-22.04" if version < 17 else "ubuntu-24.04" case "arm64": ret.os = "macos-14" pfx = f"/opt/homebrew/opt/llvm@{version}/bin/" From ff118784fa544057e3752e6c0e9291e68f8c800e Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 21:48:59 +0100 Subject: [PATCH 09/10] exclude clang18+debug from freestanding again --- .github/generate-job-matrix.py | 11 ++++++++++- .github/job_matrix.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py index b30ea573c7..e2033b9974 100644 --- a/.github/generate-job-matrix.py +++ b/.github/generate-job-matrix.py @@ -119,7 +119,12 @@ def main(): rgen = random.Random(args.seed) - collector = CombinationCollector(full_matrix) + collector = CombinationCollector( + full_matrix, + hard_excludes=lambda e: ( + e.formatting == "std::format" and not e.config.std_format_support + ), + ) match args.preset: case None: pass @@ -143,7 +148,11 @@ def main(): case "clang-tidy": collector.all_combinations(config=configs["Clang-18 (x86-64)"]) case "freestanding": + # TODO For some reason Clang-18 Debug with -ffreestanding does not pass CMakeTestCXXCompiler collector.all_combinations( + filter=lambda e: not ( + e.config.name.startswith("Clang-18") and e.build_type == "Debug" + ), config=[configs[c] for c in ["GCC-14", "Clang-18 (x86-64)"]], contracts="none", std=23, diff --git a/.github/job_matrix.py b/.github/job_matrix.py index 204be67999..0458a1e9e6 100644 --- a/.github/job_matrix.py +++ b/.github/job_matrix.py @@ -55,8 +55,14 @@ def dataclass_to_json(obj): class CombinationCollector: """Incremental builder of MatrixElements, allowing successive selection of entries.""" - def __init__(self, full_matrix: dict[str, list[typing.Any]]): + def __init__( + self, + full_matrix: dict[str, list[typing.Any]], + *, + hard_excludes: typing.Callable[[MatrixElement], bool] | None = None, + ): self.full_matrix = full_matrix + self.hard_excludes = hard_excludes self.combinations: set[MatrixElement] = set() self.per_value_counts: dict[tuple[str, typing.Any], int] = { (k, v): 0 for k, options in full_matrix.items() for v in options @@ -72,7 +78,7 @@ def _make_submatrix(self, **overrides): def _add_combination(self, e: MatrixElement): if e in self.combinations or ( - e.formatting == "std::format" and not e.config.std_format_support + self.hard_excludes is not None and self.hard_excludes(e) ): return self.combinations.add(e) From b35e241a98b1f3dc8451b7dbcd7980ea09df59f0 Mon Sep 17 00:00:00 2001 From: Yves Delley Date: Tue, 12 Nov 2024 21:58:38 +0100 Subject: [PATCH 10/10] fix clang on macos-14 (arm64) --- .github/generate-job-matrix.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/generate-job-matrix.py b/.github/generate-job-matrix.py index e2033b9974..bd14ee0000 100644 --- a/.github/generate-job-matrix.py +++ b/.github/generate-job-matrix.py @@ -26,14 +26,11 @@ def make_gcc_config(version: int) -> Configuration: def make_clang_config( version: int, platform: typing.Literal["x86-64", "arm64"] = "x86-64" ) -> Configuration: - ret = SimpleNamespace( + cfg = SimpleNamespace( name=f"Clang-{version} ({platform})", - os=None, # replaced below compiler=SimpleNamespace( type="CLANG", version=version, - cc=f"clang-{version}", - cxx=f"clang++-{version}", ), lib="libc++", cxx_modules=version >= 17, @@ -41,15 +38,18 @@ def make_clang_config( ) match platform: case "x86-64": - ret.os = "ubuntu-22.04" if version < 17 else "ubuntu-24.04" + cfg.os = "ubuntu-22.04" if version < 17 else "ubuntu-24.04" + cfg.compiler.cc = f"clang-{version}" + cfg.compiler.cxx = f"clang++-{version}" case "arm64": - ret.os = "macos-14" - pfx = f"/opt/homebrew/opt/llvm@{version}/bin/" - ret.compiler.cc = pfx + ret.compiler.cc - ret.compiler.cxx = pfx + ret.compiler.cxx + cfg.os = "macos-14" + pfx = f"/opt/homebrew/opt/llvm@{version}/bin" + cfg.compiler.cc = f"{pfx}/clang" + cfg.compiler.cxx = f"{pfx}/clang++" case _: raise KeyError(f"Unsupported platform {platform!r} for Clang") - ret.compiler = Compiler(**vars(ret.compiler)) + ret = cfg + ret.compiler = Compiler(**vars(cfg.compiler)) return Configuration(**vars(ret)) @@ -92,6 +92,8 @@ def make_msvc_config(release: str, version: int) -> Configuration: make_clang_config(ver, platform) for ver in [16, 17, 18] for platform in ["x86-64", "arm64"] + # arm64 runners are expensive; only consider one version + if ver == 18 or platform != "arm64" ] + [make_apple_clang_config(ver) for ver in [15]] + [make_msvc_config(release="14.4", version=194)]