ci improvements:

- ensure gcc and clang are tested with several optimization levels (-O3,-O2,-O1,-Os). This reveals a number of problems in gcc11,12,13 (but not 14), which should be addressed next. - use continue-on-error: false to ensure the final workflow result is red when any of its jobs fails
biojppm · Jan 12, 2025 · 40df607 · 40df607
1 parent 94a2681
commit 40df607
Show file tree

Hide file tree

Showing 32 changed files with 705 additions and 811 deletions.
diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile
@@ -19,6 +19,7 @@ test: force build
 	@git diff --exit-code $(TARGET_FILES) && \
 	  echo -e '\nPASS - No normative changes to .github/workflows/*.yml'
 
+
 DIFF_ORIG_COMMIT ?= HEAD
 diff:
 	@for s in $(SOURCE_FILES); do \
@@ -37,9 +38,11 @@ diff:
 	     ); \
 	  done
 
+
 force:
 	touch *.ys
 
+
 %.yml: %.ys $(YS) $(INPUT_FILES)
 	@if [ -f "$@" ] ; then chmod a+w $@ ; fi
 	@echo "# DO NOT EDIT - GENERATED FROM .github/workflows/$<" > $@
@@ -55,6 +58,7 @@ $(YS):
 	curl -s https://yamlscript.org/install | \
 	  BIN=1 VERSION=$(YS_VERSION) PREFIX=$(YS_PREFIX) bash
 
+
 stats:
-	@echo "ys : $$(wc -l *.ys)"
-	@echo "yml: $$(wc -l *.yml)"
+	@echo "ys : $$(wc -lm *.ys)"
+	@echo "yml: $$(wc -lm *.yml)"
diff --git a/.github/workflows/bash/benchmarks-install.bash b/.github/workflows/bash/benchmarks-install.bash
diff --git a/.github/workflows/bash/benchmarks-run.bash b/.github/workflows/bash/benchmarks-run.bash
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -83,49 +83,36 @@ jobs:
     - name: Verify vars.sh
       run: env | grep SRC
   benchmarks:
-    name: bm/c++${{matrix.std}}/${{matrix.cxx}}/${{matrix.bt}}
+    name: bm/c++${{matrix.std}}/${{matrix.comp}}/${{matrix.bt}}
     needs: gettag
-    if: |
-      (!contains(github.event.head_commit.message, 'skip all')) ||
-      (!contains(github.event.head_commit.message, 'skip benchmarks')) ||
-      contains(github.event.head_commit.message, 'only benchmarks')
-    continue-on-error: true
+    if: always()
+    continue-on-error: false
     runs-on: ${{matrix.os}}
     strategy:
       fail-fast: false
       matrix:
         include:
-        - std: 11
-          cxx: g++-10
-          bt: Release
-          os: ubuntu-20.04
-          bitlinks: static64 static32
         - std: 17
-          cxx: g++-10
-          bt: Release
-          os: ubuntu-20.04
-          bitlinks: static64 static32
-        - std: 20
-          cxx: g++-10
-          bt: Release
-          os: ubuntu-20.04
-          bitlinks: static64 static32
+          comp: gcc14
+          os: ubuntu-24.04
+          cmkflags: -DCMAKE_CXX_COMPILER=g++-14     -DCMAKE_C_COMPILER=gcc-14
         - std: 17
-          cxx: vs2019
-          bt: Release
-          os: windows-2019
-          bitlinks: static64 static32
-        - std: 20
-          cxx: vs2019
-          bt: Release
+          comp: clang18
+          os: ubuntu-24.04
+          cmkflags: -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18
+        - std: 17
+          comp: vs2019
           os: windows-2019
-          bitlinks: static64 static32
+          cmkflags: -G 'Visual Studio 16 2019' -A x64
         - std: 17
-          cxx: xcode
-          xcver: 15
-          bt: Release
+          comp: vs2022
+          os: windows-2022
+          cmkflags: -G 'Visual Studio 17 2022' -A x64
+        - std: 17
+          comp: xcode15
           os: macos-13
-          bitlinks: static64
+          xcver: 15
+          cmkflags: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64
     env:
       BM: 'ON'
       STD: ${{matrix.std}}
@@ -182,47 +169,21 @@ jobs:
         pip install -v -r ext/c4core/cmake/bm-xp/requirements.txt
         python -c 'import munch ; print("ok!") ; exit(0)'
         echo $?
-    - name: set build env vars
-      run: |
-        echo "NUM_JOBS_BUILD=1" >> $GITHUB_ENV
-        echo "CMAKE_FLAGS=\"-DPython_EXECUTABLE=$(which python)\"" >> $GITHUB_ENV
-    - name: check build env vars
+    - name: configure-------------------------------------------------
       run: |
-        echo NUM_JOBS_BUILD=$NUM_JOBS_BUILD
-        echo "CMAKE_FLAGS=\"$CMAKE_FLAGS\""
-    - name: shared64-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test shared64
-    - name: shared64-build
-      run: source .github/setenv.sh && c4_build_target shared64 ryml-bm-build
-    - name: shared64-run
-      run: source .github/setenv.sh && c4_run_target shared64 ryml-bm-run
-    - name: shared64-plot
-      run: source .github/setenv.sh && c4_run_target shared64 ryml-bm-plot
-    - name: static64-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test static64
-    - name: static64-build
-      run: source .github/setenv.sh && c4_build_target static64 ryml-bm-build
-    - name: static64-run
-      run: source .github/setenv.sh && c4_run_target static64 ryml-bm-run
-    - name: static64-plot
-      run: source .github/setenv.sh && c4_run_target static64 ryml-bm-plot
-    - name: static32-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test static32
-    - name: static32-build
-      run: source .github/setenv.sh && c4_build_target static32 ryml-bm-build
-    - name: static32-run
-      run: source .github/setenv.sh && c4_run_target static32 ryml-bm-run
-    - name: static32-plot
-      run: source .github/setenv.sh && c4_run_target static32 ryml-bm-plot
-    - name: shared32-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test shared32
-    - name: shared32-build
-      run: source .github/setenv.sh && c4_build_target shared32 ryml-bm-build
-    - name: shared32-run
-      run: source .github/setenv.sh && c4_run_target shared32 ryml-bm-run
-    - name: shared32-plot
-      run: source .github/setenv.sh && c4_run_target shared32 ryml-bm-plot
-    - name: gather benchmark results
+        cmake -S . -B build \
+            ${{matrix.cmkflags}} \
+            -DPython_EXECUTABLE="$(which python)" \
+            -DC4_CXX_STANDARD=${{matrix.std}} \
+            -DRYML_CXX_STANDARD=${{matrix.std}} \
+            -DRYML_BUILD_BENCHMARKS=ON
+    - name: build
+      run: cmake --build build --config Release --parallel --verbose --target ryml-bm-build
+    - name: run
+      run: cmake --build build --config Release --verbose --target ryml-bm-run
+    - name: plot
+      run: cmake --build build --config Release --verbose --target ryml-bm-plot
+    - name: gather benchmark results-------------------------------------------
       run: |
         set -x
         source vars.sh

diff --git a/.github/workflows/benchmarks.ys b/.github/workflows/benchmarks.ys
@@ -11,23 +11,19 @@ jobs:
     - :: version-discover
 
   benchmarks:
-    name: bm/c++${{matrix.std}}/${{matrix.cxx}}/${{matrix.bt}}
+    name: bm/c++${{matrix.std}}/${{matrix.comp}}/${{matrix.bt}}
     needs: gettag
-    :: setup-job('benchmarks')
+    :: setup-job('benchmarks' 'benchmarks')
     runs-on: ${{matrix.os}}
     strategy:
       fail-fast: false
       matrix:
-        item =: \({:std %1, :cxx %2, :bt 'Release', :os %3, :bitlinks 'static64 static32'})
         include:
-        - ! item(11 'g++-10' 'ubuntu-20.04')
-        - ! item(17 'g++-10' 'ubuntu-20.04')
-        - ! item(20 'g++-10' 'ubuntu-20.04')
-        - ! item(17 'vs2019' 'windows-2019')
-        - ! item(20 'vs2019' 'windows-2019')
-       #- ! item(17 'vs2019' 'windows-2022')
-       #- ! item(20 'vs2019' 'windows-2022')
-        - {std: 17, cxx: xcode, xcver: 15, bt: Release, os: macos-13, bitlinks: static64}
+        - {std: 17, comp: gcc14  , os: ubuntu-24.04, cmkflags: -DCMAKE_CXX_COMPILER=g++-14     -DCMAKE_C_COMPILER=gcc-14}
+        - {std: 17, comp: clang18, os: ubuntu-24.04, cmkflags: -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18}
+        - {std: 17, comp: vs2019 , os: windows-2019, cmkflags: -G 'Visual Studio 16 2019' -A x64}
+        - {std: 17, comp: vs2022 , os: windows-2022, cmkflags: -G 'Visual Studio 17 2022' -A x64}
+        - {std: 17, comp: xcode15, os: macos-13, xcver: 15, cmkflags: -G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64}
     env:: -{'BM' 'ON'} + load('share/env.yaml')
     steps:
     - :: checkout-action
@@ -40,49 +36,41 @@ jobs:
       uses: actions/setup-python@v5
       with: { python-version: '3.10' }
     - name: install benchmark plotting dependencies
-      run:: bash('benchmarks-install')
-    - name: set build env vars
       run: |
-        echo "NUM_JOBS_BUILD=1" >> $GITHUB_ENV
-        echo "CMAKE_FLAGS=\"-DPython_EXECUTABLE=$(which python)\"" >> $GITHUB_ENV
-    - name: check build env vars
+        which python
+        which pip
+        python --version
+        pip --version
+        pip install -v -r ext/c4core/cmake/bm-xp/requirements.txt
+        python -c 'import munch ; print("ok!") ; exit(0)'
+        echo $?
+    - name: configure-------------------------------------------------
       run: |
-        echo NUM_JOBS_BUILD=$NUM_JOBS_BUILD
-        echo "CMAKE_FLAGS=\"$CMAKE_FLAGS\""
-    - name: shared64-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test shared64
-    - name: shared64-build
-      run: source .github/setenv.sh && c4_build_target shared64 ryml-bm-build
-    - name: shared64-run
-      run: source .github/setenv.sh && c4_run_target shared64 ryml-bm-run
-    - name: shared64-plot
-      run: source .github/setenv.sh && c4_run_target shared64 ryml-bm-plot
-    - name: static64-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test static64
-    - name: static64-build
-      run: source .github/setenv.sh && c4_build_target static64 ryml-bm-build
-    - name: static64-run
-      run: source .github/setenv.sh && c4_run_target static64 ryml-bm-run
-    - name: static64-plot
-      run: source .github/setenv.sh && c4_run_target static64 ryml-bm-plot
-    - name: static32-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test static32
-    - name: static32-build
-      run: source .github/setenv.sh && c4_build_target static32 ryml-bm-build
-    - name: static32-run
-      run: source .github/setenv.sh && c4_run_target static32 ryml-bm-run
-    - name: static32-plot
-      run: source .github/setenv.sh && c4_run_target static32 ryml-bm-plot
-    - name: shared32-configure-------------------------------------------------
-      run: source .github/setenv.sh && c4_cfg_test shared32
-    - name: shared32-build
-      run: source .github/setenv.sh && c4_build_target shared32 ryml-bm-build
-    - name: shared32-run
-      run: source .github/setenv.sh && c4_run_target shared32 ryml-bm-run
-    - name: shared32-plot
-      run: source .github/setenv.sh && c4_run_target shared32 ryml-bm-plot
-    - name: gather benchmark results
-      run:: bash('benchmarks-run')
+        cmake -S . -B build \
+            ${{matrix.cmkflags}} \
+            -DPython_EXECUTABLE="$(which python)" \
+            -DC4_CXX_STANDARD=${{matrix.std}} \
+            -DRYML_CXX_STANDARD=${{matrix.std}} \
+            -DRYML_BUILD_BENCHMARKS=ON
+    - name: build
+      run: cmake --build build --config Release --parallel --verbose --target ryml-bm-build
+    - name: run
+      run: cmake --build build --config Release --verbose --target ryml-bm-run
+    - name: plot
+      run: cmake --build build --config Release --verbose --target ryml-bm-plot
+    - name: gather benchmark results-------------------------------------------
+      run: |
+        set -x
+        source vars.sh
+        echo SRC_TAG=$SRC_TAG
+        echo SRC_VERSION=$SRC_VERSION
+        desc=$SRC_TAG
+        for bl in ${{matrix.bitlinks}} ; do
+          dst=$(echo ryml-benchmark_results/$desc/x86_64/${{matrix.cxx}}-${{matrix.bt}}-c++${{matrix.std}}-$bl | sed 's:++-:xx:g' | sed 's:+:x:g')
+          mkdir -p $dst
+          find build -name bm-results
+          mv -vf build/$bl/bm/bm-results/* $dst/.
+        done
     - name: upload benchmark result artifacts
       uses: actions/upload-artifact@v3
       with: