Release workflow (#68)

* add release workflow

* build hdf5 manually and link against that static lib

* update cache key to reflect whether it contains HDF5

* build static libs on macos too and link to those

* ensure GNU gcc is used, not clang

* dont error if linking fails (this is to avoid gfrotran, which already exsists)

* only need to link gcc and g++

* pin homebrew gcc-13 version

* use read writeable MCFOST_INSTALL path for any user

* fix gcc install

* gfortran issue

* single dependency action

* do release as part of main workflow

* xattr -dr com.apple.quarantine mcfost [skip ci]

* link statically with libgfortran and libquadmath

* command 'xattr -dr com.apple.quarantine mcfost' does nothing, needs to be
done by user once file is downloaded

* skip test step for releases. Master is protected so test should already
have passed

* use binary stored in artifact and upload to release

upload for linux and macos, always using ifort build

* only run release job on release

* debug file permissions

* fix bug caused by tar running as root

* separate tar fix into its own step

* remove debugging line

* link statically with libgfortran and libquadmath

* libstdc++ should come after libvoro++ when linking

some linkers will only search for functions from left to right in the
link line. This commit ensures the code compiles regardless of the linker
behaviour

see e.g.:
https://www.linuxtopia.org/online_books/an_introduction_to_gcc/gccintro_18.html

* fix incorrect artifact name

* always run test, even for release

* run mcfost -h to make sure it runs in a clean env

* remove redundant release check

* temporarily disable test

* chmod +x mcfost

* fix path

* fix

* just use gfortran for testing

* ./

* use actual os for release runner

* missing $

* correct filename

* Revert "just use gfortran for testing"

This reverts commit f406bf8.

* Revert "temporarily disable test"

This reverts commit f6060c0.

* fix filenames

* use march=native, unless it's a release

* nicer name for build+test action

* run CI for all PRs, not just on master

---------

Co-authored-by: Conrad Chan <conradchan@hotmail.com.au>
Co-authored-by: Conrad Chan <8309215+conradtchan@users.noreply.github.com>

.github/actions/dependencies/action.yml

@@ -25,7 +25,7 @@ runs:
         chmod +x l_fortran-compiler_p_2023.2.1.8_offline.sh
         ./l_fortran-compiler_p_2023.2.1.8_offline.sh -a --silent --eula accept
         echo "INTEL_PATH=/home/runner/intel/oneapi" >> "$GITHUB_ENV"
-        sudo chown root /bin/tar && sudo chmod u+s /bin/tar
+        sudo chown root /bin/tar && sudo chmod u+s /bin/tar # tar needs to run as root to extract when restoring the cache
 
     - name: fix gcc issue on the runner (GNU / macos)
       if: matrix.compiler == 'gfortran' && matrix.os == 'macos'
@@ -58,6 +58,11 @@ runs:
           ${{ env.MCFOST_INSTALL }}/lib/${{ matrix.compiler }}
         key: mcfost-deps-${{ runner.os }}-${{ matrix.compiler }}-${{ hashFiles('lib/install.sh') }}
 
+    - name: unset setuid bit for tar if cache not found
+      if: matrix.compiler == 'ifort' && matrix.os == 'linux' && steps.cache-deps.outputs.cache-hit != 'true'
+      shell: bash
+      run: sudo chmod u-s /bin/tar
+
     # Only do this (lengthy) setup if dependency cache not found
     - name: prepare mcfost environment
       if: ${{ steps.cache-deps.outputs.cache-hit != 'true' }}
@@ -66,4 +71,4 @@ runs:
       run: |
         [ ! "$SETVARS_COMPLETED" == 1 ] && test -f "$INTEL_PATH"/setvars.sh && . "$INTEL_PATH"/setvars.sh
         sudo chmod o+w /usr/local/include /usr/local/lib
-        ./install.sh
+        ./install.sh

.github/actions/release/action.yml

@@ -0,0 +1,17 @@
+name: "release"
+description: "release"
+
+runs:
+  using: "composite"
+  steps:
+    - name: prepare tarball
+      working-directory: src
+      shell: bash
+      run: tar -cvzf mcfost.tar.gz mcfost
+
+    - name: Add binary to release
+      uses: shogo82148/actions-upload-release-asset@v1.6.6
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: src/mcfost.tar.gz
+        asset_name: ${{ format('mcfost_{0}-{1}.tar.gz', runner.os, runner.arch) }}

.github/workflows/test-suite.yml

@@ -4,7 +4,9 @@ on:
   push:
     branches: [ master ]
   pull_request:
-    branches: [ master ]
+  release:
+    types:
+      - published
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -22,6 +24,7 @@ jobs:
         os: [linux, macos]
         compiler: [gfortran, ifort]
         openmp: [yes, no]
+        release: [yes, no]
 
         include:
           # Associate specific os versions with our os tags
@@ -36,6 +39,7 @@ jobs:
             compiler: gfortran
             container: "quay.io/pypa/manylinux_2_28_x86_64"
 
+    name: build+test (${{ matrix.os }}, ${{ matrix.compiler }}, openmp=${{ matrix.openmp }}, release=${{ matrix.release }})
     runs-on: ${{ matrix.os-version }}
     container: ${{ matrix.container }}
     env:
@@ -57,17 +61,18 @@ jobs:
         shell: bash -e {0}
         run: |
           [ ! "$SETVARS_COMPLETED" == 1 ] && test -f "$INTEL_PATH"/setvars.sh && . "$INTEL_PATH"/setvars.sh
-          make static=yes all 
+          make release=${{ matrix.release }} static=yes all
 
       - name: test
         uses: ./.github/actions/test
 
       # Saving the binaries from the macos builds since the runner is the same for both compilers -> might be more consistent
-      - name: save binary for benchmarking
-        if: matrix.os == 'macos' && matrix.openmp == 'yes'
+      # Only benchmark+release the 'release' builds, never the 'march=native' builds
+      - name: save binary for benchmarking and release
+        if: matrix.openmp == 'yes' && matrix.release == 'yes'
         uses: actions/upload-artifact@v3
         with:
-          name: mcfost-binary-${{ matrix.compiler }}
+          name: mcfost-binary-${{ matrix.os }}-${{ matrix.compiler }}
           path: src/mcfost
 
   benchmark:
@@ -86,8 +91,8 @@ jobs:
 
       - name: restore binary artifact
         uses: actions/download-artifact@v3
-        with: 
-          name: mcfost-binary-${{ matrix.compiler }}
+        with:
+          name: mcfost-binary-macos-${{ matrix.compiler }}
           path: src/
 
       - name: benchmark
@@ -96,3 +101,41 @@ jobs:
       - name: report benchmark results
         run: |
           echo ${{ matrix.compiler }} $(cat benchmark-${{ matrix.compiler }}.tim)
+
+
+  release:
+    needs: build-and-test
+    strategy:
+      matrix:
+        os: [linux, macos]
+
+        include:
+          # Associate specific os versions with our os tags
+          - os: linux
+            os-version: ubuntu-22.04
+          - os: macos
+            os-version: macos-12
+
+    runs-on: ${{ matrix.os-version }}
+    if: ${{ github.event_name == 'release' }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: restore binary artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: mcfost-binary-${{ matrix.os }}-ifort # use ifort for binary release
+          path: src/
+
+      - name: simple check to make sure the binary runs in a clean environment
+        shell: bash
+        working-directory: src
+        env:
+          MCFOST_UTILS: ${{ github.workspace }}/utils
+        run: |
+          chmod +x mcfost
+          ./mcfost -h
+
+      - name: upload binary
+        uses: ./.github/actions/release

src/Makefile

@@ -1,6 +1,9 @@
 #------------------------------------------------------------------------------#
 # Makefile for Monte Carlo radiative transfer code MCFOST
 #------------------------------------------------------------------------------#
+
+IS_DARWIN := $(shell uname | tr '[a-z]' '[A-Z]' 2>&1 | grep -c DARWIN)
+
 # Linux compiling server
 LINUX_SERVER=gadi
 IPAG_SERVER=ipag-nfs
@@ -25,14 +28,14 @@ ifeq ($(ifx), yes)
     SYSTEM=ifx
 endif
 
+ifeq ($(release), yes)
+    RELEASE=yes
+endif
+
 ifndef OPENMP
     OPENMP=yes
 endif
 
-# Forcing static flags when compiling the releases
-release : LDFLAGS+= $(STATIC_FLAGS)
-release_mac : LDFLAGS+= $(STATIC_FLAGS)
-
 # faster compilation
 ifeq ($(dev), yes)
     DEV=yes
@@ -83,15 +86,15 @@ endif
 # System dependent compiler flags
 
 ifeq ($(SYSTEM), ifort)
-    ifeq ($(shell uname | tr '[a-z]' '[A-Z]' 2>&1 | grep -c DARWIN),1)
-        ARCH= -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic
-	STATIC_FLAGS= -static-intel -qopenmp-link static
+    STATIC_FLAGS = -static-intel -qopenmp-link static
+    ifeq ($(IS_DARWIN),1)
+        ARCH = -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic
     else
-        ARCH= -axSSE2,SSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512
-	STATIC_FLAGS= -static-intel -qopenmp-link static -static-libstdc++
+        ARCH = -axSSE2,SSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512
+        STATIC_FLAGS += -static-libstdc++
     endif
     FC= ifort
-    FFLAGS= -fpp -O3 -no-prec-div -fp-model fast=2 -traceback $(ARCH)
+    FFLAGS= -fpp -O3 -no-prec-div -fp-model fast=2 -traceback
     DBLFLAG= -r8
     DEBUGFLAG= -check all -C -g -fpe0 -traceback -no-ftz \
                -warn uninitialized -warn unused -warn truncated_source
@@ -100,22 +103,22 @@ ifeq ($(SYSTEM), ifort)
     IPOFLAGS= -ipo
     CC= icc
     CXX=icpc -std=c++11
-    CCFLAGS= -O3 -no-prec-div -fp-model fast=2 -traceback $(ARCH)
+    CCFLAGS= -O3 -no-prec-div -fp-model fast=2 -traceback
     COMPFLAGS= -fopenmp
     LIBCXX= -cxxlib
     LIBS= $(LIB)/ifort
 endif
 
 ifeq ($(SYSTEM), ifx)
-    ifeq ($(shell uname | tr '[a-z]' '[A-Z]' 2>&1 | grep -c DARWIN),1)
-        ARCH= -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic
-        STATIC_FLAGS= -static-intel -qopenmp-link static
+    STATIC_FLAGS = -static-intel -qopenmp-link static
+    ifeq ($(IS_DARWIN),1)
+        ARCH = -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2 -mmacosx-version-min=10.13 -mdynamic-no-pic
     else
-        ARCH= -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512
-        STATIC_FLAGS= -static-intel -qopenmp-link static -static-libstdc++
+        ARCH = -axSSSE3,SSE4.1,SSE4.2,AVX,CORE-AVX2,CORE-AVX512
+        STATIC_FLAGS += -static-libstdc++
     endif
     FC= ifx
-    FFLAGS= -fpp -O3 -fp-model fast=2 -traceback $(ARCH)
+    FFLAGS= -fpp -O3 -fp-model fast=2 -traceback
     DBLFLAG= -r8
     DEBUGFLAG= -check all -C -g -fpe0 -traceback -no-ftz \
                -warn uninitialized -warn unused -warn truncated_source
@@ -124,7 +127,7 @@ ifeq ($(SYSTEM), ifx)
     IPOFLAGS= -ipo
     CC= icx
     CXX=icpx -std=c++11
-    CCFLAGS= -O3 -fp-model fast=2 -traceback $(ARCH)
+    CCFLAGS= -O3 -fp-model fast=2 -traceback
     COMPFLAGS= -qopenmp
     LIBCXX= -cxxlib
     LIBS= $(LIB)/ifx
@@ -134,7 +137,7 @@ ifeq ($(SYSTEM), gfortran)
     FC= gfortran
     FFLAGS+= -m64 -cpp -O3 -funroll-loops -ffinite-math-only -fno-trapping-math \
 	     -ffree-line-length-132 -fbacktrace
-    STATIC_FLAGS= -static-libgcc -static-libstdc++
+    STATIC_FLAGS= -static-libgcc -static-libstdc++ -static-libgfortran -static-libquadmath
     DBLFLAG= -fdefault-real-8 -fdefault-double-8
     DEBUGFLAG= -g -fbounds-check -Wunused-variable -Wunused-function -Wuninitialized -fbacktrace
     KNOWN_SYSTEM= yes
@@ -153,6 +156,16 @@ endif
 #------------------------------------------------------------------------------#
 # Set other optional flags depending on system settings
 
+# Force static flags when compiling the releases, and march=native if not a release
+ifeq ($(RELEASE), yes)
+    STATIC = yes
+    FFLAGS += $(ARCH)
+    CCFLAGS += $(ARCH)
+else
+    FFLAGS += -march=native
+    CCFLAGS += -march=native
+endif
+
 ifeq ($(DEBUG), yes)
     FFLAGS+= ${DEBUGFLAG}
     FFLAGS:= $(FFLAGS:-O3=-O0)
@@ -164,7 +177,7 @@ endif
 
 
 ifeq ($(DEV), yes)
-   FFLAGS:= $(FFLAGS:-O3=-O1)
+    FFLAGS:= $(FFLAGS:-O3=-O1)
 endif
 
 ifeq ($(OPENMP), yes)
@@ -302,47 +315,11 @@ mcfost2prodimo :  mcfost mcfost2prodimo.o
 #------------------------------------------------------------------------------#
 # Release
 
-.PHONY : release release_mac release_utils fosti dmz wardlaw
-
-release : main
-	- git tag -d release
-	git tag release
-	git push github
-	git push github --tags --force
-	mkdir -p $(MCFOST_INSTALL)/bin/
-	\cp -f mcfost $(MCFOST_INSTALL)/bin/
-	rm -rf web
-	mkdir web web/linux web/mac
-	cp mcfost mcfost_update
-	tar czvf mcfost_bin.tgz mcfost_update ; rm -rf mcfost_update
-	openssl sha1 mcfost_bin.tgz | awk '{print $$2}' > web/mac/mcfost.sha1
-	mv mcfost_bin.tgz web/mac/
-	ssh $(LINUX_SERVER) "cd mcfost/src && git pull origin master && git fetch --tags --force && git checkout release && export MCFOST_GIT=1 && export MAKEFLAGS= && make clean mcfost static=yes && git checkout master && \cp -f mcfost ~/mcfost_cigri && \cp -f mcfost mcfost_update && rm -f mcfost.sha1 && tar czf mcfost_bin.tgz mcfost_update --remove-files && sha1sum mcfost_bin.tgz  | awk '{print $$1}' > mcfost.sha1"
-	scp $(LINUX_SERVER):mcfost/src/mcfost_bin.tgz web/linux/
-	scp $(LINUX_SERVER):mcfost/src/mcfost.sha1 web/linux/
-	cp `grep mcfost_version mcfost_env.f90 | awk '{print "ref"$$6".para ref"$$6"_multi.para ref"$$6"_3D.para"}' ` web
-	grep mcfost_release mcfost_env.f90 | awk '{print $$6}' | sed s/\"//g > web/version.txt
-	git log --pretty=format:"%cd  %h %d %s" --date=short  --graph | sed s/"dmz\/master, phare\/master, github\/master, bitbucket\/master"// | sed s/"()"// | sed s/", ,"/","/ > web/history.txt
-	rsync -Pur  web/* $(IPAG_SERVER):/user/publicdir/pintec/mcfost
-	ssh $(IPAG_SERVER) "cd  /user/publicdir/pintec/mcfost/mac ; tar xzf mcfost_bin.tgz ; mv mcfost_update mcfost ; rm -f mcfost.tgz ; tar czf mcfost.tgz mcfost --remove-files ; cd  /user/publicdir/pintec/mcfost/linux ; tar xzf mcfost_bin.tgz ; mv mcfost_update mcfost ; rm -f mcfost.tgz ; tar czf mcfost.tgz mcfost --remove-files"
+.PHONY : release_utils fosti dmz wardlaw
 
 release_doc:
 	pushd . ; cd ../docs ; make html ; cd _build ; tar czf html.tgz html ; scp -r html.tgz $(IPAG_SERVER):/user/publicdir/pintec/mcfost/docs ; ssh $(IPAG_SERVER) "cd /user/publicdir/pintec/mcfost/docs ; tar xzf html.tgz ; rm -f html.tgz" ; popd
 
-release_mac : main
-	\cp -f mcfost $(MCFOST_INSTALL)/bin/
-	rm -rf web
-	mkdir web web/mac
-	cp mcfost mcfost_update
-	tar czvf mcfost_bin.tgz mcfost_update ; rm -rf mcfost_update
-	openssl sha1 mcfost_bin.tgz | awk '{print $$2}' > web/mac/mcfost.sha1
-	mv mcfost_bin.tgz web/mac/
-	cp `grep mcfost_version mcfost_env.f90 | awk '{print "ref"$$6".para ref"$$6"_multi.para ref"$$6"_3D.para"}' ` web
-	grep mcfost_release mcfost_env.f90 | awk '{print $$6}' | sed s/\"//g > web/version.txt
-	git log --pretty=format:"%cd  %h  %d %s" --date=short --graph --branches > web/history.txt
-	rsync -Pur -r web/* $(IPAG_SERVER):/user/publicdir/pintec/mcfost
-	ssh $(IPAG_SERVER) "cd  /user/publicdir/pintec/mcfost/mac ; tar xzf mcfost_bin.tgz ; mv mcfost_update mcfost ; rm -f mcfost.tgz ; tar czf mcfost.tgz mcfost --remove-files"
-
 release_utils :
 	tar czvf mcfost_utils.tgz -C../utils .
 	openssl sha1 mcfost_utils.tgz | awk '{print $2}' > mcfost_utils.sha1