Merge branch '957-suppression-numba' into 'master'

957-suppression-numba

Closes #957

See merge request 3d/cars-park/cars!801

.gitattributes

@@ -0,0 +1,3 @@
+docs/ export-ignore
+tests/ export-ignore
+tutorials/ export-ignore

.github/workflows/cars-ci.yml

@@ -24,10 +24,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip setuptools
+        python -m pip install --upgrade pip meson-python meson ninja setuptools_scm setuptools wheel pybind11
         pip install --upgrade cython numpy
-        pip install --upgrade numba
-        pip install .[dev,docs,notebook,pandora_mccnn]
+        pip install --no-build-isolation --editable .[dev,docs,notebook,pandora_mccnn]
         pip list
     - name: Unit Tests
       run: |

Makefile

@@ -31,9 +31,6 @@ endif
 CHECK_NUMPY = $(shell ${CARS_VENV}/bin/python -m pip list|grep numpy)
 CHECK_FIONA = $(shell ${CARS_VENV}/bin/python -m pip list|grep Fiona)
 CHECK_RASTERIO = $(shell ${CARS_VENV}/bin/python -m pip list|grep rasterio)
-CHECK_TBB = $(shell ${CARS_VENV}/bin/python -m pip list|grep tbb)
-CHECK_NUMBA = $(shell ${CARS_VENV}/bin/python -m pip list|grep numba)
-TBB_VERSION_SETUP = $(shell cat setup.cfg | grep tbb |cut -d = -f 3 | cut -d ' ' -f 1)
 
 # Check Docker
 CHECK_DOCKER = $(shell docker -v)
@@ -47,65 +44,28 @@ CARS_VERSION_MIN =$(shell echo ${CARS_VERSION} | cut -d . -f 1,2,3)
 .PHONY: help
 help: ## this help
 	@echo "      CARS MAKE HELP  LOGLEVEL=${LOGLEVEL}"
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'| sort
+	@grep -E '^[a-zA-Z_/-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'| sort
 
 ## Install section
 
 .PHONY: venv
-venv: ## create virtualenv in CARS_VENV directory if not exists
+venv: ## create virtualenv in CARS_VENV directory if it doesn't exist already
 	@test -d ${CARS_VENV} || python3 -m venv ${CARS_VENV}
-	@${CARS_VENV}/bin/python -m pip install --upgrade pip setuptools # no check to upgrade each time
+	@${CARS_VENV}/bin/python -m pip install --upgrade pip meson-python meson ninja setuptools_scm setuptools wheel pybind11 # no check to upgrade each time
 	@touch ${CARS_VENV}/bin/activate
 
-.PHONY: install-deps
-install-deps: venv ## install python libs
+.PHONY: install/deps
+install/deps: venv ## install python libs
 	@[ "${CHECK_NUMPY}" ] ||${CARS_VENV}/bin/python -m pip install --upgrade numpy
-	@[ "${CHECK_TBB}" ] ||${CARS_VENV}/bin/python -m pip install tbb==$(TBB_VERSION_SETUP)
-	@[ "${CHECK_NUMBA}" ] ||${CARS_VENV}/bin/python -m pip install --upgrade numba
 
-.PHONY: install-deps-gdal
-install-deps-gdal: install-deps ## create an healthy python environment for GDAL/ proj
+.PHONY: install/deps-gdal
+install/deps-gdal: install/deps ## create an healthy python environment for GDAL/ proj
 	@[ "${CHECK_FIONA}" ] ||${CARS_VENV}/bin/python -m pip install --no-binary fiona fiona
 	@[ "${CHECK_RASTERIO}" ] ||${CARS_VENV}/bin/python -m pip install --no-binary rasterio rasterio
 
-.PHONY: install
-install: install-deps ## install cars (not editable) with dev, docs, notebook dependencies
-	@test -f ${CARS_VENV}/bin/cars || ${CARS_VENV}/bin/pip install .[dev,docs,notebook]
-	@test -f .git/hooks/pre-commit || echo "  Install pre-commit hook"
-	@test -f .git/hooks/pre-commit || ${CARS_VENV}/bin/pre-commit install -t pre-commit
-	@test -f .git/hooks/pre-push || ${CARS_VENV}/bin/pre-commit install -t pre-push
-	@echo "CARS ${CARS_VERSION} installed in dev mode in virtualenv ${CARS_VENV}"
-	@echo "CARS venv usage: source ${CARS_VENV}/bin/activate; cars -h"
-
-.PHONY: install-gdal
-install-gdal: install-deps-gdal ## install cars (not editable) with dev, docs, notebook dependencies
-	@test -f ${CARS_VENV}/bin/cars || ${CARS_VENV}/bin/pip install .[dev,docs,notebook,pandora_mccnn]
-	@test -f .git/hooks/pre-commit || echo "  Install pre-commit hook"
-	@test -f .git/hooks/pre-commit || ${CARS_VENV}/bin/pre-commit install -t pre-commit
-	@test -f .git/hooks/pre-push || ${CARS_VENV}/bin/pre-commit install -t pre-push
-	@echo "CARS ${CARS_VERSION} installed in dev mode in virtualenv ${CARS_VENV}"
-	@echo "CARS venv usage: source ${CARS_VENV}/bin/activate; cars -h"
-
-.PHONY: install-gdal-dev
-install-gdal-dev: install-deps-gdal ## install cars dev (editable) with dev, docs, notebook dependencies
-	@test -f ${CARS_VENV}/bin/cars || ${CARS_VENV}/bin/pip install .[dev,docs,notebook,pandora_mccnn]
-	@test -f .git/hooks/pre-commit || echo "  Install pre-commit hook"
-	@test -f .git/hooks/pre-commit || ${CARS_VENV}/bin/pre-commit install -t pre-commit
-	@test -f .git/hooks/pre-push || ${CARS_VENV}/bin/pre-commit install -t pre-push
-	@echo "CARS ${CARS_VERSION} installed in dev mode in virtualenv ${CARS_VENV}"
-	@echo "CARS venv usage: source ${CARS_VENV}/bin/activate; cars -h"
-
-.PHONY: install-pandora-mccnn
-install-pandora-mccnn: install-deps  ## install cars (not editable) with dev, docs, notebook dependencies
-	@test -f ${CARS_VENV}/bin/cars || ${CARS_VENV}/bin/pip install .[dev,docs,notebook,pandora_mccnn]
-	@test -f .git/hooks/pre-commit || echo "  Install pre-commit hook"
-	@test -f .git/hooks/pre-commit || ${CARS_VENV}/bin/pre-commit install -t pre-commit
-	@test -f .git/hooks/pre-push || ${CARS_VENV}/bin/pre-commit install -t pre-push
-	@echo "CARS ${CARS_VERSION} installed in virtualenv ${CARS_VENV}"
-
-.PHONY: install-dev
-install-dev: install-deps ## install cars in dev editable mode (pip install -e .) without recompiling rasterio, fiona
-	@test -f ${CARS_VENV}/bin/cars || ${CARS_VENV}/bin/pip install -e .[dev,docs,notebook]
+.PHONY: install/deps
+install: install/deps ## install cars in dev editable mode (pip install --no-build-isolation -e .) without recompiling rasterio and fiona
+	@test -f ${CARS_VENV}/bin/cars || source ${CARS_VENV}/bin/activate; pip install --no-build-isolation --editable .[dev,docs,notebook]
 	@test -f .git/hooks/pre-commit || echo "  Install pre-commit hook"
 	@test -f .git/hooks/pre-commit || ${CARS_VENV}/bin/pre-commit install -t pre-commit
 	@test -f .git/hooks/pre-push || ${CARS_VENV}/bin/pre-commit install -t pre-push
@@ -115,31 +75,23 @@ install-dev: install-deps ## install cars in dev editable mode (pip install -e .
 ## Test section
 
 .PHONY: test
-test: ## run unit tests without SLURM cluster + coverage html
-	@${CARS_VENV}/bin/pytest -m "unit_tests and not pbs_cluster_tests" -o log_cli=true -o log_cli_level=${LOGLEVEL} --cov-config=.coveragerc --cov-report html --cov
+test: ## run unit tests + coverage html
+	@${CARS_VENV}/bin/pytest -m "unit_tests" -o log_cli=true -o log_cli_level=${LOGLEVEL} --cov-config=.coveragerc --cov-report html --cov
 
-.PHONY: test-ci
-test-ci: ## run unit and pbs tests + coverage for cars-ci
-	@${CARS_VENV}/bin/pytest -m "unit_tests or pbs_cluster_tests" --durations=0 --log-date-format="%Y-%m-%d %H:%M:%S" --log-format="%(asctime)s [%(levelname)8s] (%(filename)s:%(lineno)s) : %(message)s"  -o log_cli=true -o log_cli_level=${LOGLEVEL} --junitxml=pytest-report.xml --cov-config=.coveragerc --cov-report xml --cov
+.PHONY: test/ci
+test/ci: ## run unit and pbs tests + coverage for cars-ci
+	@${CARS_VENV}/bin/pytest -m "unit_tests" --durations=0 --log-date-format="%Y-%m-%d %H:%M:%S" --log-format="%(asctime)s [%(levelname)8s] (%(filename)s:%(lineno)s) : %(message)s"  -o log_cli=true -o log_cli_level=${LOGLEVEL} --junitxml=pytest-report.xml --cov-config=.coveragerc --cov-report xml --cov
 
-.PHONY: test-end2end
-test-end2end: ## run end2end tests only
+.PHONY: test/end2end
+test/end2end: ## run end2end tests only
 	@${CARS_VENV}/bin/pytest -m "end2end_tests" -o log_cli=true -o log_cli_level=${LOGLEVEL}
 
-.PHONY: test-unit
-test-unit: ## run unit tests only
+.PHONY: test/unit
+test/unit: ## run unit tests only
 	@${CARS_VENV}/bin/pytest -m "unit_tests" -o log_cli=true -o log_cli_level=${LOGLEVEL}
 
-.PHONY: test-pbs-cluster
-test-pbs-cluster: ## run pbs cluster tests only
-	@${CARS_VENV}/bin/pytest -m "pbs_cluster_tests" -o log_cli=true -o log_cli_level=${LOGLEVEL}
-
-.PHONY: test-slurm-cluster
-test-slurm-cluster: ## run slurm cluster tests only
-	@${CARS_VENV}/bin/pytest -m "slurm_cluster_tests" -o log_cli=true -o log_cli_level=${LOGLEVEL}
-
-.PHONY: test-notebook
-test-notebook: ## run notebook tests only
+.PHONY: test/notebook
+test/notebook: ## run notebook tests only
 	@${CARS_VENV}/bin/pytest -m "notebook_tests" -o log_cli=true -o log_cli_level=${LOGLEVEL}
 
 ## Code quality, linting section
@@ -205,20 +157,20 @@ notebook: ## install Jupyter notebook kernel with venv and cars install
 # Dev section
 
 .PHONY: dev
-dev: install-dev docs notebook ## install CARS in dev mode : install-dev, notebook and docs
+dev: install docs notebook ## install CARS, compile docs, prepare notebooks
 
 ## Docker section
 
-.PHONY: docker-deps
-docker-deps: ## Check and build docker image cnes/cars-deps
+.PHONY: docker/deps
+docker/deps: ## Check and build docker image cnes/cars-deps
 	@@[ "${CHECK_DOCKER}" ] || ( echo ">> docker not found"; exit 1 )
 	@docker pull hadolint/hadolint
 	@echo "Check Dockerfile with hadolint"
 	@docker run --rm -i hadolint/hadolint < Dockerfile
 	@echo "Hadolint ok"
 
 .PHONY: docker
-docker: docker-deps ## Check and build docker image cnes/cars
+docker: docker/deps ## Check and build docker image cnes/cars
 # Set docker options like --build-arg
 ifndef DOCKER_OPTIONS
 	@docker build -t cnes/cars -t cnes/cars:latest . -f Dockerfile
@@ -230,37 +182,37 @@ endif
 ## Clean section
 
 .PHONY: clean
-clean: clean-venv clean-build clean-precommit clean-pyc clean-test clean-docs clean-notebook clean-dask ## remove all build, test, coverage and Python artifacts
+clean: clean/venv clean/build clean/precommit clean/pyc clean/test clean/docs clean/notebook clean/dask ## remove all build, test, coverage and Python artifacts
 
-.PHONY: clean-venv
-clean-venv:
+.PHONY: clean/venv
+clean/venv:
 	@echo "+ $@"
 	@echo ${CARS_VENV}
 	@rm -rf ${CARS_VENV}
 
-.PHONY: clean-build
-clean-build:
+.PHONY: clean/build
+clean/build:
 	@echo "+ $@"
 	@rm -fr build/
 	@rm -fr dist/
 	@rm -fr .eggs/
 	@find . -name '*.egg-info' -exec rm -fr {} +
 	@find . -name '*.egg' -exec rm -f {} +
 
-.PHONY: clean-precommit
-clean-precommit:
+.PHONY: clean/precommit
+clean/precommit:
 	@rm -f .git/hooks/pre-commit
 	@rm -f .git/hooks/pre-push
 
-.PHONY: clean-pyc
-clean-pyc:
+.PHONY: clean/pyc
+clean/pyc:
 	@echo "+ $@"
 	@find . -type f -name "*.py[co]" -exec rm -fr {} +
 	@find . -type d -name "__pycache__" -exec rm -fr {} +
 	@find . -name '*~' -exec rm -fr {} +
 
-.PHONY: clean-test
-clean-test:
+.PHONY: clean/test
+clean/test:
 	@echo "+ $@"
 	@rm -fr .tox/
 	@rm -f .coverage
@@ -272,36 +224,35 @@ clean-test:
 	@rm -f pylint-report.txt
 	@rm -f debug.log
 
-.PHONY: clean-docs
-clean-docs:
+.PHONY: clean/docs
+clean/docs:
 	@echo "+ $@"
 	@rm -rf docs/build/
 	@rm -rf docs/source/api_reference/
 
-.PHONY: clean-notebook
-clean-notebook:
+.PHONY: clean/notebook
+clean/notebook:
 	@echo "+ $@"
 	@find . -type d -name ".ipynb_checkpoints" -exec rm -fr {} +
 
-.PHONY: clean-dask
-clean-dask:
+.PHONY: clean/dask
+clean/dask:
 	@echo "+ $@"
 	@find . -type d -name "dask-worker-space" -exec rm -fr {} +
 
-
-.PHONY: clean-docker
-clean-docker: ## clean docker image
+.PHONY: docker/clean
+docker/clean: ## clean docker image
 	@@[ "${CHECK_DOCKER}" ] || ( echo ">> docker not found"; exit 1 )
 	@echo "Clean Docker images cars ${CARS_VERSION_MIN}"
 	@docker image rm -f cnes/cars:${CARS_VERSION_MIN}
 	@docker image rm -f cnes/cars:latest
 
 
-.PHONY: profile-memory-report
-profile-memory-report: ## build report after execution of cars with profiling memray mode (report biggest  memory occupation for each application), indicate the output_result directory file
+.PHONY: profile/memory-report
+profile/memory-report: ## build report after execution of cars with profiling memray mode (report biggest  memory occupation for each application), indicate the output_result directory file
 	@for file in $(wildcard ./$(filter-out $@,$(MAKECMDGOALS))/profiling/memray/*.bin); do echo $$file && ${CARS_VENV}/bin/memray tree -b 10 $$file; done;
 
-.PHONY: profile-memory-all
-profile-memory-all: ## memory profiling at master orchestrator level (not only at worker level) with cars CLI command, uses config.json as input (please use sequential orchestrator mode and desactive profiling)
+.PHONY: profile/memory-all
+profile/memory-all: ## memory profiling at master orchestrator level (not only at worker level) with cars CLI command, uses config.json as input (please use sequential orchestrator mode and desactive profiling)
 	@${CARS_VENV}/bin/memray run -o memray.result.bin ${CARS_VENV}/bin/cars $(wildcard ./$(filter-out $@,$(MAKECMDGOALS)))
 	@${CARS_VENV}/bin/memray tree -b 50 memray.result.bin

NOTICE

@@ -27,11 +27,6 @@ Copyright (c) 1997-2018 OpenMP ARB Corporation
 Website: https://www.openmp.org/
 License: "GNU GPL v3.0 + GNU GCC Runtime Library Exception" (GCC) and "NCSA Open Source License" (Clang)
 
-TBB: flexible C++ library for parallelism applications
-Copyright (c) 2016-2021 Intel Corporation
-Website: https://github.com/oneapi-src/oneTBB
-Licence: Apache v2.0.
-
 PANDORA: Stereo pipeline which produces disparity map from a pair of images. 
 Copyright (c) 2020, Centre National d'Etudes Spatiales (CNES).
 Website: https://github.com/CNES/Pandora
@@ -84,11 +79,6 @@ Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com>
 Website: https://setuptools.readthedocs.io/
 License: MIT.
 
-Numba: NumPy aware dynamic Python compiler using LLVM
-Copyright (c) 2012, Anaconda, Inc.
-Website: https://numba.pydata.org/
-License: BSD 2 Clause.
-
 Xarray: N-D labeled arrays and datasets in Python
 Copyright 2014-2019, xarray Developers
 Website: http://xarray.pydata.org

cars/__init__.py

@@ -56,7 +56,6 @@
 os.environ["SHARELOC_NUMBA_PARALLEL"] = str(False)
 os.environ["OMP_NUM_THREADS"] = "1"
 os.environ["MKL_NUM_THREADS"] = "1"
-os.environ["NUMBA_NUM_THREADS"] = "1"
 os.environ["GDAL_NUM_THREADS"] = "1"
 
 # Limit GDAL cache per worker to 500MB

cars/applications/dense_match_filling/cpp/dense_match_filling_cpp.py

@@ -0,0 +1,72 @@
+"""
+this module contains the headers of the dense_match_filling_cpp module.
+"""
+
+# pylint: skip-file
+
+from typing import Tuple
+
+import numpy as np
+
+
+def fill_disp_pandora(
+    disp: np.ndarray, msk_fill_disp: np.ndarray, nb_directions: int
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Interpolation of the left disparity map to fill holes.
+    Interpolate invalid pixel by finding the nearest correct pixels in
+    8/16 different directions and use the median of their disparities.
+    ?bontar, J., & LeCun, Y. (2016). Stereo matching by training
+    a convolutional neural network to compare image
+    patches. The journal of machine learning research, 17(1), 2287-2318.
+    HIRSCHMULLER, Heiko. Stereo processing by semiglobal matching
+    and mutual information.
+    IEEE Transactions on pattern analysis and machine intelligence,
+    2007, vol. 30, no 2, p. 328-341.
+
+    Copied/adapted fct from pandora/validation/interpolated_disparity.py
+
+    :param disp: disparity map
+    :type disp: 2D np.array (row, col)
+    :param msk_fill_disp: validity mask
+    :type msk_fill_disp: 2D np.array (row, col)
+    :param nb_directions: nb directions to explore
+    :type nb_directions: integer
+
+    :return: the interpolate left disparity map,
+        with the validity mask update :
+    :rtype: tuple(2D np.array (row, col), 2D np.array (row, col))
+    """
+    return None, None
+
+
+def find_valid_neighbors(
+    dirs: np.ndarray,
+    disp: np.ndarray,
+    valid: np.ndarray,
+    row: int,
+    col: int,
+    nb_directions: int,
+):
+    """
+    Find valid neighbors along directions
+
+    Copied/adapted fct from pandora/validation/interpolated_disparity.py
+
+    :param dirs: directions
+    :type dirs: 2D np.array (row, col)
+    :param disp: disparity map
+    :type disp: 2D np.array (row, col)
+    :param valid: validity mask
+    :type valid: 2D np.array (row, col)
+    :param row: row current value
+    :type row: int
+    :param col: col current value
+    :type col: int
+    :param nb_directions: nb directions to explore
+    :type nb_directions: int
+
+    :return: valid neighbors
+    :rtype: 2D np.array
+    """
+    ...