New update stack2 (#1246)

* source code maintenance and stack update

* move data files to LFS

* update travis file

* update environment-dev (conda) and stack

* remove pytables usage and other defunct code in archiving

* make build options uniform across platforms; update proteus_tutorial

* enable cache on travis for hashdist_src/_bld

* update proteus_tutorials submodule testing

* added gitattributes so lfs works properly

* update air-water-vv submodule

* move apt dependencies inside hashdist linux container in travis

* turn off coverage on parallel test

* re-arrange cache and coverage steps

* add --cov-append to prevent coverage overwrite

* removing proteus build from path before cache upload

* removing unused triangle module

* excluding cython files from coverage

* fix missing return value in quad mesh generation in MeshTools

* fix missing return value in uWindow in WaveTools

* enable compiling on macos using 2 cores

.coveragerc

@@ -0,0 +1,13 @@
+[run]
+#plugins = Cython.Coverage # not working for now; excluding cython source below for now
+source = proteus
+omit =
+  */config/*
+  */test_utils/*
+  */tests/*
+  #cythonized modules, not currently showing coverage but tests are running
+  */proteus/WaveTools.py
+  */proteus/BoundaryConditions.py
+  */proteus/SpatialTools.py
+  */proteus/mprans/BoundaryConditions.py
+  */proteus/mprans/SpatialTools.py

.travis.yml

@@ -1,69 +1,80 @@
-addons:
-  apt:
-    packages:
-    - gfortran
-    - libffi-dev
-    - libssl-dev
-    - python-lzma
-    - python3
-    - doxygen
-    - graphviz
-
-cache:
-  - timeout: 1000
+os: linux
+dist: focal
+language: python
+python: 3.9
 
 notifications:
   email:
     on_success: change
     on_failure: always
 
-after_success:
-  - conda install codecov
-  - codecov
-
 jobs:
   include:
   - stage: test
     name: "hashdist linux"
     os: linux
-    dist: bionic
+    dist: focal
     language: python
-    python: 2.7
+    python: 3.9
+    addons:
+      apt:
+        packages:
+        - gfortran
+        - libffi-dev
+        - libssl-dev
+        - python2 # for hit
+        - doxygen
+        - graphviz
+        - ca-certificates
+        - libsqlite3-dev
     env: TEST_PROFILE="proteus-hashdist"
+    cache:
+      - directories:
+        - hashdist_bld
+        - hashdist_src
     install:
+      - export SSL_CERT_DIR=/etc/ssl/certs
       - git lfs pull
       - ./stack/hit/bin/hit init-home
-      - ./stack/hit/bin/hit remote add http://levant.hrwallingford.com/hashdist_src --objects="source"
-      - ./stack/hit/bin/hit remote add http://levant.hrwallingford.com/hashdist_stack --objects="build"
+      - ./stack/hit/bin/hit remote add https://proteus.cee.lsu.edu/hashdist_src --objects="source"
+      - ./stack/hit/bin/hit remote add https://proteus.cee.lsu.edu/hashdist_travis_focal --objects="build"
       - make stack/default.yaml
+      - mkdir -p hashdist_bld
+      - mkdir -p hashdist_src
       - pushd $HOME
+      - rm -rf .hashdist/bld
+      - ln -s /home/travis/build/erdc/proteus/hashdist_bld .hashdist/bld
       - mkdir -p hashdist_src
-      - mkdir -p hashdist_bld
-      - rm -rf .hashdist/src .hashdist/bld
-      - ln -s $HOME/hashdist_src .hashdist/src
-      - ln -s $HOME/hashdist_bld .hashdist/bld
+      - rm -rf .hashdist/src
+      - ln -s /home/travis/build/erdc/proteus/hashdist_src .hashdist/src
       - popd
       - pushd stack
       - echo $PWD
       - ls -l
-      - ./hit/bin/hit build -j 2 -v default.yaml
+      - PYTHONHTTPSVERIFY=0 ./hit/bin/hit build -j 2 -v default.yaml
       - popd
       - export PATHSAVE=$PATH
       - export PATH=$PWD/linux/bin:$PATH
+      - export LD_LIBRARY_PATHSAVE=$LD_LIBRARY_PATH
       - export LD_LIBRARY_PATH=$PWD/linux/lib:$LD_LIBRARY_PATH
-      - PROTEUS_OPT="-w -g0 -O2 -UNDEBUG" FC=gfortran CC=mpicc CXX=mpicxx make develop N=2
-      - export SSL_CERT_DIR=/etc/ssl/certs
+      - PROTEUS_OPT="-g0 -O2 -DNDEBUG" FC=gfortran CC=mpicc CXX=mpicxx make develop N=2
     script:
-      - PATH=./linux/bin:$PATH MPLBACKEND=Agg py.test -n 1 --dist=loadfile --forked -v proteus/tests --ignore proteus/tests/POD --ignore proteus/tests/solver_tests/test_nse_RANS2P_step.py --cov=proteus
-      - PATH=./linux/bin:$PATH MPLBACKEND=Agg py.test -n 1 --dist=loadfile --forked -v air-water-vv/Tests/1st_set --cov=proteus
-      - PATH=./linux/bin:$PATH MPLBACKEND=Agg py.test -n 1 --dist=loadfile --forked -v air-water-vv/Tests/2nd_set --cov=proteus
-      - PATH=./linux/bin:$PATH MPLBACKEND=Agg mpiexec -np 2 pytest -v proteus/tests/FSI
+      - export MPLBACKEND=Agg
+      - py.test -n 1 --forked -v proteus/tests --ignore proteus/tests/POD --ignore proteus/tests/solver_tests/test_nse_RANS2P_step.py --cov --cov-config=.coveragerc
+      - py.test -n 1 --forked -v air-water-vv/Tests/1st_set --cov --cov-append --cov-config=.coveragerc
+      - py.test -n 1 --forked -v air-water-vv/Tests/2nd_set --cov --cov-append --cov-config=.coveragerc
+      - mpiexec -np 2 pytest -v proteus/tests/FSI #can't  do --cov in parallel for now
+      - export PATH=$PATHSAVE
+      - export LD_LIBRARY_PATH=$LD_LIBRARY_PATHSAVE
+    after_success:
+      - pip install codecov
+      - codecov
   - stage: test
     name: "conda linux"
     os: linux
-    dist: xenial
+    dist: focal
     language: python
-    python: 3.7
+    python: 3.9
     env: TEST_PROFILE="proteus-conda"
     install:
       - git lfs pull
@@ -77,13 +88,13 @@ jobs:
       - conda info -a
       - conda env create -f environment-dev.yml
       - conda activate proteus-dev
-      - make develop-conda N=1
+      - PROTEUS_OPT="-g0 -O2 -DNDEBUG" FC=gfortran CC=mpicc CXX=mpicxx make develop-conda N=2
     script:
       - export MPLBACKEND="AGG"
-      - py.test -n 1 --dist=loadfile --forked -v proteus/tests --ignore proteus/tests/POD --ignore proteus/tests/solver_tests/test_nse_RANS2P_step.py --cov=proteus
-      - MPLBACKEND=Agg py.test -n 1 --dist=loadfile --forked -v air-water-vv/Tests/1st_set
-      - MPLBACKEND=Agg py.test -n 1 --dist=loadfile --forked -v air-water-vv/Tests/2nd_set
-      - MPLBACKEND=Agg mpiexec -np 2 pytest -v proteus/tests/FSI
+      - py.test -n 1 --forked -v proteus/tests --ignore proteus/tests/POD --ignore proteus/tests/solver_tests/test_nse_RANS2P_step.py
+      - py.test -n 1 --forked -v air-water-vv/Tests/1st_set
+      - py.test -n 1 --forked -v air-water-vv/Tests/2nd_set
+      - mpiexec -np 2 pytest -v proteus/tests/FSI
     deploy:
       provider: script
       script: bash ./scripts/deploy.sh docs
@@ -93,7 +104,9 @@ jobs:
   - stage: test
     name: "conda osx"
     os: osx
-    osx_image: xcode11.3
+    osx_image: xcode12.2
+    language: ruby
+    python: 2.6.6
     env: TEST_PROFILE="proteus-conda-osx"
     install:
       - wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
@@ -106,11 +119,11 @@ jobs:
       - conda env create -f environment-dev.yml
       - conda activate proteus-dev
       - pip install gmsh
-      - pip install -v -e .
+      - PROTEUS_OPT="-g0 -O2 -DNDEBUG" FC=gfortran CC=mpicc CXX=mpicxx make develop-conda N=2
     script:
       - export MPLBACKEND="AGG"
       - git lfs install
       - git lfs pull
-      - py.test -n 1 --dist=loadfile --forked -v proteus/tests --ignore proteus/tests/solver_tests --ignore proteus/tests/POD --ignore proteus/tests/MeshAdaptPUMI --ignore=proteus/tests/matrix_constructor --ignore=proteus/tests/MoveMeshMonitor --ignore-glob='proteus/tests/periodic/*test_periodic.py' --ignore proteus/tests/solver_tests/test_nse_RANS2P_step.py --ignore proteus/tests/poisson_2d --cov=proteus
-      - PATH=./linux/bin:$PATH MPLBACKEND=Agg py.test -n 1 --dist=loadfile --forked -v air-water-vv/Tests/1st_set --cov=proteus
-      - PATH=./linux/bin:$PATH MPLBACKEND=Agg py.test -n 1 --dist=loadfile --forked -v air-water-vv/Tests/2nd_set --ignore air-water-vv/Tests/2nd_set/test_randomWaves.py --cov=proteus
+      - py.test -n 1 --forked -v proteus/tests --ignore proteus/tests/solver_tests --ignore proteus/tests/POD --ignore=proteus/tests/MoveMeshMonitor
+      - py.test -n 1 --forked -v air-water-vv/Tests/1st_set
+      - py.test -n 1 --forked -v air-water-vv/Tests/2nd_set --ignore air-water-vv/Tests/2nd_set/test_randomWaves.py

Makefile

@@ -118,7 +118,7 @@ stack/hit/bin/hit:
 	cd stack && git submodule update --init
 	@echo "Adding source cache if not done already"
 	-./stack/hit/bin/hit init-home
-	-./stack/hit/bin/hit remote add http://levant.hrwallingford.com/hashdist_src --objects="source"
+	-./stack/hit/bin/hit remote add https://proteus.cee.lsu.edu/hashdist_src --objects="source"
 
 stack:
 	@echo "Updating stack submodule"
@@ -131,7 +131,7 @@ air-water-vv:
 bld_cache: stack/hit/bin/hit
 	@echo "Trying to add build cache for your arch"
 	HASHSTACK_BLD = $(shell lsb_release -ir | python3 -c "import sys; rel=dict((k.split(':')[0].split()[0],k.split(':')[1].strip().replace('.','_').lower()) for k in sys.stdin.readlines()); print('{Distributor}_{Release}'.format(**rel))")
-	./stack/hit/bin/hit remote add http://levant.hrwallingford.com/hashdist_${HASHSTACK_BLD} --objects="build"
+	./stack/hit/bin/hit remote add https://proteus.cee.lsu.edu/hashdist_${HASHSTACK_BLD} --objects="build"
 
 cygwin_bootstrap.done: stack/scripts/setup_cygstack.py stack/scripts/cygstack.txt
 	python3 stack/scripts/setup_cygstack.py stack/scripts/cygstack.txt
@@ -197,6 +197,7 @@ develop: ${PROTEUS_PREFIX}/bin/proteus_env.sh stack/default.yaml ${PROTEUS_PREFI
 	@echo "Installing development version"
 	@echo "************************"
 	$(call show_info)
+	${PROTEUS_ENV} pip --use-feature=2020-resolver install recordtype py2gmsh pytest pytest-xdist pytest-cov nose tables future
 	${PROTEUS_ENV} ${PROTEUS_DEVELOP_BUILD_CMD}
 	${PROTEUS_ENV} ${PROTEUS_DEVELOP_CMD}
 	@echo "************************"

codecov.yml

@@ -0,0 +1,3 @@
+ignore:
+  - "proteus/tests"
+  - "proteus/scripts"

environment-dev.yml

@@ -3,55 +3,57 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - pip =20.0.2
-  - compilers =1.0.4
+  - pip =21.0.1
+  - compilers =1.1.1
   - make
-  - setuptools =45.2.0
-  - cython =0.29.15
+  - setuptools =49.6.0
+  - cython =0.29.21
   - daetk =1.0.1
-  - hdf5 =1.10.5=*mpich*
-  - h5py =2.10.0=*mpich*
+  - hdf5 =1.10.6=*mpich*
+  - h5py =3.1.0=*mpich*
   - metis =5.1.0
   - mpich =3.3.2
-  - numpy =1.14.6
-  - openblas =0.3.7=*_7
+  - mpich-mpicc =3.3.2
+  - mpich-mpicxx =3.3.2
+  - numpy =1.20.1
+  - openblas =0.3.10
   - parmetis =4.0.3
-  - petsc4py =3.12.0
-  - petsc =3.12.4
+  - petsc4py =3.13.0
+  - petsc =3.13.6
   - scorec =2.2.2
-  - superlu =5.2.1
+  - superlu =5.2.2
   - superlu_dist =6.2.0
   - triangle =1.6
-  - pychrono =5.0.0
-  - gmsh
-  - matplotlib =3.1.3
+  - pychrono =6.0.0
+  - gmsh =4.6.0
+  - matplotlib =3.3.4
   - mpi4py =3.0.3
   - nose
   - pytables =3.6.1
   - pytest
   - pytest-cov
   - pytest-xdist
-  - scipy =1.4.1
-  - tetgen =1.5.1
-  - ncurses =6.1
-  - python =3.7.6
+  - scipy =1.5.3
+  - tetgen =1.6.0
+  - ncurses =6.2
+  - python =3.9.1
   - future =0.18.2
-  - ipyparallel =6.2.4
-  - pillow =7.0.0
+  - ipyparallel =6.3.0
+  - pillow =8.1.0
   - recordtype =1.3
-  - xtensor-python
+  - xtensor-python = 0.25.1
   - git-lfs
-  - eigen =3.3.8
+  - eigen =3.3.9
   - zoltan =3.83
   - hypre =2.18.2
   - mumps-mpi =5.2.1
-  - numexpr =2.7.1
-  - ptscotch =6.0.8
+  - numexpr =2.7.2
+  - ptscotch =6.0.9
   - scalapack =2.0.2
-  - tbb =2019.9
-  - zipp =3.0.0
-  - parso =0.6.1
-  - pycparser =2.19
-  - ipython =7.12.0
-  - decorator =4.4.1
-  - cffi =1.13.2
+  - tbb =2020.2
+  - zipp =3.4.0
+  - parso =0.8.1
+  - pycparser =2.20
+  - ipython =7.20.0
+  - decorator =4.4.2
+  - cffi =1.14.5