From 82a66bbd8ae7394920e8774dbe604d1c497509b5 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 15:04:33 +0100 Subject: [PATCH 01/35] demo workflow --- .github/workflows/github-actions-demo.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/github-actions-demo.yaml diff --git a/.github/workflows/github-actions-demo.yaml b/.github/workflows/github-actions-demo.yaml new file mode 100644 index 00000000..e007988b --- /dev/null +++ b/.github/workflows/github-actions-demo.yaml @@ -0,0 +1,18 @@ +name: GitHub Actions Demo +run-name: ${{ github.actor }} is testing out GitHub Actions 🚀 +on: [push] +jobs: + Explore-GitHub-Actions: + runs-on: ubuntu-latest + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v3 + - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." + - run: echo "🖥️ The workflow is now ready to test your code on the runner." + - name: List files in the repository + run: | + ls ${{ github.workspace }} + - run: echo "🍏 This job's status is ${{ job.status }}." From 24db82314dd4a3233bf5a824929f6f2d50e13547 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 16:45:43 +0100 Subject: [PATCH 02/35] try pr workflow --- .github/workflows/pr.yaml | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/pr.yaml diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 00000000..23103745 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,44 @@ +name: Python package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Set up venv + run: | + python3 -m venv env + source env/bin/activate + python3 -m pip install --upgrade pip setuptools + echo "Python path: $(which python3)" + echo "Python version: $(python3 --version)" + echo "pip version: $(python3 -m pip --version)" + echo "pip version: $(python3 -m setuptools --version)" + - name: Install dependencies + run: | + source env/bin/activate + echo "Python path: $(which python3)" + echo "Python version: $(python3 --version)" + echo "pip version: $(python3 -m pip --version)" + echo "pip version: $(python3 -m setuptools --version)" + python3 -m pip install .[devel] + - name: Test with pytest + run: | + source env/bin/activate + echo "Python path: $(which python3)" + echo "Python version: $(python3 --version)" + echo "pip version: $(python3 -m pip --version)" + echo "pip version: $(python3 -m setuptools --version)" + python3 -m pytest --cov=fklearn tests/ + codecov From df1a7e766cff76aae030f94e7386bfbcda0ee92c Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 16:48:13 +0100 Subject: [PATCH 03/35] pin ubuntu 20.04 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 23103745..739f1d24 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -5,7 +5,7 @@ on: [push] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] From f2341e7442fdb5803243b22ee2bdac1e75978c07 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 16:55:43 +0100 Subject: [PATCH 04/35] scikit-learn fix --- .github/workflows/pr.yaml | 3 --- requirements.txt | 2 +- src/fklearn/training/unsupervised.py | 10 +++++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 739f1d24..bac3df1b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,14 +24,12 @@ jobs: echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - echo "pip version: $(python3 -m setuptools --version)" - name: Install dependencies run: | source env/bin/activate echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - echo "pip version: $(python3 -m setuptools --version)" python3 -m pip install .[devel] - name: Test with pytest run: | @@ -39,6 +37,5 @@ jobs: echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - echo "pip version: $(python3 -m setuptools --version)" python3 -m pytest --cov=fklearn tests/ codecov diff --git a/requirements.txt b/requirements.txt index 69c22256..b33f771c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ joblib>=0.13.2,<2 numpy>=1.16.4,<2 pandas>=0.24.1,<2 -scikit-learn>=0.21.2,<0.24.0 +scikit-learn>=0.21.2,<0.25.0 statsmodels>=0.9.0,<1 toolz>=0.9.0,<1 diff --git a/src/fklearn/training/unsupervised.py b/src/fklearn/training/unsupervised.py index cf7acbc7..df8ef528 100644 --- a/src/fklearn/training/unsupervised.py +++ b/src/fklearn/training/unsupervised.py @@ -42,13 +42,17 @@ def isolation_forest_learner(df: pd.DataFrame, If True, treats all columns in `df` with name pattern fklearn_feat__col==val` as feature columns. """ - default_params = {"n_jobs": -1, "random_state": 1729, "contamination": 0.1, "behaviour": "new"} + model = IsolationForest() + + default_params = {"n_jobs": -1, "random_state": 1729, "contamination": 0.1} + # Remove this when we stop supporting scikit-learn<0.24 as this param is deprecated + if "behaviour" in model.get_params(): + default_params["behaviour"] = "new" params = default_params if not params else merge(default_params, params) + model.set_params(**params) features = features if not encode_extra_cols else expand_features_encoded(df, features) - model = IsolationForest() - model.set_params(**params) model.fit(df[features].values) def p(new_df: pd.DataFrame) -> pd.DataFrame: From 23f3cd8bff0e6cbe991b5500167ceb5f2ea2ae09 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 17:00:19 +0100 Subject: [PATCH 05/35] remove 3.11 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bac3df1b..f8fa64ff 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v3 From b444f507346e97546ebdb48592ba979ab91d47f8 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 17:12:49 +0100 Subject: [PATCH 06/35] try another coverage syntax --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f8fa64ff..86bc654f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -37,5 +37,5 @@ jobs: echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - python3 -m pytest --cov=fklearn tests/ + python3 -m pytest tests --cov=src/ codecov From 4701443eb4d7047847182230fa475964cd243c2d Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 17:13:33 +0100 Subject: [PATCH 07/35] remove 3.10 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 86bc654f..8924ebf3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9"] steps: - uses: actions/checkout@v3 From d9a10c273cdd448d17ca578e81417669178b10d3 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 17:22:04 +0100 Subject: [PATCH 08/35] missing -e install --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8924ebf3..381212e1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -30,7 +30,7 @@ jobs: echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - python3 -m pip install .[devel] + python3 -m pip install -e .[devel] - name: Test with pytest run: | source env/bin/activate From 2a8a099e84fc01749d41acde00a5ff4d9298c96b Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 19:18:28 +0100 Subject: [PATCH 09/35] try another coverage syntax --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 381212e1..7cc52b81 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -37,5 +37,5 @@ jobs: echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - python3 -m pytest tests --cov=src/ + python3 -m pytest --cov-config=.coveragerc --cov=src/ tests codecov From f84bdf92f4e2668629b6ae8de7df13da02cda4fb Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 19:33:01 +0100 Subject: [PATCH 10/35] Another coverage syntax --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7cc52b81..ee3f7d46 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -37,5 +37,5 @@ jobs: echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - python3 -m pytest --cov-config=.coveragerc --cov=src/ tests + python3 -m pytest --cov-config=.coveragerc tests codecov From 36c5b976dd8a1ac68bb41e9d9d96d5729806b160 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 20 Mar 2023 19:47:34 +0100 Subject: [PATCH 11/35] Yet another coverage syntax --- .coveragerc | 5 ----- .github/workflows/pr.yaml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 35dd4024..c7edcbb3 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,3 @@ -[run] -parallel = True -concurrency = multiprocessing -branch = True - [report] exclude_lines = raise * diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ee3f7d46..b7857eee 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -37,5 +37,5 @@ jobs: echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" - python3 -m pytest --cov-config=.coveragerc tests + python3 -m pytest --cov-config=.coveragerc --cov=fklearn tests/ codecov From 6deab0b420f41879567a7c47374e0527323eff79 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Tue, 21 Mar 2023 15:05:45 +0100 Subject: [PATCH 12/35] pin coverage --- requirements_test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_test.txt b/requirements_test.txt index 4f604782..5a351e2e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,5 +2,6 @@ pytest>=4.2.1,<7 pytest-cov>=2.6.1,<3 pytest-xdist>=1.26.1,<3 mypy>=0.670,<1 +coverage<5 codecov>=2.0,<3 hypothesis>=5.5.4,<7 From ea841c78bc79ec1a91b4b3ebdd824f27710b8a4b Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 14:10:07 +0100 Subject: [PATCH 13/35] remove coverage bot --- .github/workflows/pr.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b7857eee..26efae43 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,6 +18,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Set up venv run: | + set -euxo pipefail python3 -m venv env source env/bin/activate python3 -m pip install --upgrade pip setuptools @@ -26,16 +27,13 @@ jobs: echo "pip version: $(python3 -m pip --version)" - name: Install dependencies run: | + set -euxo pipefail source env/bin/activate - echo "Python path: $(which python3)" - echo "Python version: $(python3 --version)" - echo "pip version: $(python3 -m pip --version)" + python3 -m pip install -e .[devel] - name: Test with pytest run: | + set -euxo pipefail source env/bin/activate - echo "Python path: $(which python3)" - echo "Python version: $(python3 --version)" - echo "pip version: $(python3 -m pip --version)" - python3 -m pytest --cov-config=.coveragerc --cov=fklearn tests/ - codecov + + python3 -m pytest --cov=fklearn tests/ From 80e7e1fb9b0b5b68ef6461b3e6b2cda02cf5a83f Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 14:25:12 +0100 Subject: [PATCH 14/35] test fail below coverage --- .github/workflows/pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 26efae43..5aeba9b9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,9 +19,11 @@ jobs: - name: Set up venv run: | set -euxo pipefail + python3 -m venv env source env/bin/activate python3 -m pip install --upgrade pip setuptools + echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" @@ -36,4 +38,4 @@ jobs: set -euxo pipefail source env/bin/activate - python3 -m pytest --cov=fklearn tests/ + python3 -m pytest --cov-fail-under=97 --cov=fklearn tests/ From 88c97660f9b15c18ceb5a862f10fe3e7099f83ee Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 14:28:48 +0100 Subject: [PATCH 15/35] test wrong test --- .github/workflows/pr.yaml | 4 ++-- tests/training/test_classification.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5aeba9b9..821143e6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,7 +23,7 @@ jobs: python3 -m venv env source env/bin/activate python3 -m pip install --upgrade pip setuptools - + echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" @@ -38,4 +38,4 @@ jobs: set -euxo pipefail source env/bin/activate - python3 -m pytest --cov-fail-under=97 --cov=fklearn tests/ + python3 -m pytest --cov-fail-under=95 --cov=fklearn tests/ diff --git a/tests/training/test_classification.py b/tests/training/test_classification.py index e7da8793..6c6be9f0 100644 --- a/tests/training/test_classification.py +++ b/tests/training/test_classification.py @@ -59,7 +59,7 @@ def test_logistic_classification_learner(): assert Counter(expected_col_train) == Counter(pred_train.columns.tolist()) assert Counter(expected_col_test) == Counter(pred_test.columns.tolist()) - assert pred_test.prediction.max() < 1 + assert pred_test.prediction.max() > 1 assert pred_test.prediction.min() > 0 assert (pred_test.columns == pred_train.columns).all() From 9e90d28a527920994a9dac91c2f18897b14914f1 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 14:34:03 +0100 Subject: [PATCH 16/35] Revert "test wrong test" This reverts commit 88c97660f9b15c18ceb5a862f10fe3e7099f83ee. --- .github/workflows/pr.yaml | 4 ++-- tests/training/test_classification.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 821143e6..5aeba9b9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,7 +23,7 @@ jobs: python3 -m venv env source env/bin/activate python3 -m pip install --upgrade pip setuptools - + echo "Python path: $(which python3)" echo "Python version: $(python3 --version)" echo "pip version: $(python3 -m pip --version)" @@ -38,4 +38,4 @@ jobs: set -euxo pipefail source env/bin/activate - python3 -m pytest --cov-fail-under=95 --cov=fklearn tests/ + python3 -m pytest --cov-fail-under=97 --cov=fklearn tests/ diff --git a/tests/training/test_classification.py b/tests/training/test_classification.py index 6c6be9f0..e7da8793 100644 --- a/tests/training/test_classification.py +++ b/tests/training/test_classification.py @@ -59,7 +59,7 @@ def test_logistic_classification_learner(): assert Counter(expected_col_train) == Counter(pred_train.columns.tolist()) assert Counter(expected_col_test) == Counter(pred_test.columns.tolist()) - assert pred_test.prediction.max() > 1 + assert pred_test.prediction.max() < 1 assert pred_test.prediction.min() > 0 assert (pred_test.columns == pred_train.columns).all() From f4b9fd3b363ce58b4a275013d691672189311ed8 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 14:59:27 +0100 Subject: [PATCH 17/35] Add pip check, type check and linter --- .github/workflows/pr.yaml | 40 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5aeba9b9..6c2ef0b2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,9 +1,31 @@ -name: Python package +name: PR check suite on: [push] jobs: - build: + + linter: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: + run: | + set -euxo pipefail + python3 -m pip install -q flake8==3.8.4 + python3 -m flake8 \ + --ignore=E731,W503 \ + --filename=*.py \ + --exclude=__init__.py \ + --show-source \ + --statistics \ + --max-line-length=120 \ + src/ tests/ + + test-suite: runs-on: ubuntu-20.04 strategy: @@ -33,9 +55,21 @@ jobs: source env/bin/activate python3 -m pip install -e .[devel] + - name: Pip check + run: | + set -euxo pipefail + source env/bin/activate + + pip check + - name: Type check + run: | + set -euxo pipefail + source env/bin/activate + + python3 -m mypy src tests --config mypy.ini - name: Test with pytest run: | set -euxo pipefail source env/bin/activate - python3 -m pytest --cov-fail-under=97 --cov=fklearn tests/ + python3 -m pytest --cov-fail-under=95 --cov=fklearn tests/ From fcd326d1573916cc0e483286d9625b86038a3496 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 15:07:39 +0100 Subject: [PATCH 18/35] Add type annotation to dict in isolation forest learner --- src/fklearn/training/unsupervised.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fklearn/training/unsupervised.py b/src/fklearn/training/unsupervised.py index df8ef528..4e597e03 100644 --- a/src/fklearn/training/unsupervised.py +++ b/src/fklearn/training/unsupervised.py @@ -44,7 +44,7 @@ def isolation_forest_learner(df: pd.DataFrame, model = IsolationForest() - default_params = {"n_jobs": -1, "random_state": 1729, "contamination": 0.1} + default_params: Dict[str, Any] = {"n_jobs": -1, "random_state": 1729, "contamination": 0.1} # Remove this when we stop supporting scikit-learn<0.24 as this param is deprecated if "behaviour" in model.get_params(): default_params["behaviour"] = "new" From 02404df86ed79e8065d435b429e70251396a3442 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 15:19:26 +0100 Subject: [PATCH 19/35] Try implicit optional config for mypy --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index ec200257..bef05eac 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,6 @@ [mypy] ignore_missing_imports = True +implicit_optional = True [mypy-fklearn.*] disallow_untyped_defs = True From 1470a928308bc9ee4152707206fe42d5515d3a45 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 15:31:34 +0100 Subject: [PATCH 20/35] more type annotation --- src/fklearn/training/classification.py | 4 ++-- src/fklearn/training/transformation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fklearn/training/classification.py b/src/fklearn/training/classification.py index 86b8258d..ac1dc4a0 100644 --- a/src/fklearn/training/classification.py +++ b/src/fklearn/training/classification.py @@ -235,7 +235,7 @@ def p(new_df: pd.DataFrame, apply_shap: bool = False) -> pd.DataFrame: @curry def _get_catboost_shap_values(df: pd.DataFrame, cbr: Any, features: List, target: str, - weights: List, cat_features: List) -> np.array: + weights: List, cat_features: List) -> np.ndarray: """ Auxiliar method to allow us to get shap values for Catboost multiclass models @@ -519,7 +519,7 @@ def lgbm_classification_learner(df: pd.DataFrame, keep_training_booster: bool = False, callbacks: Optional[List[Callable]] = None, dataset_init_score: Optional[Union[ - List, List[List], np.array, pd.Series, pd.DataFrame] + List, List[List], np.ndarray, pd.Series, pd.DataFrame] ] = None) -> LearnerReturnType: """ Fits an LGBM classifier to the dataset. diff --git a/src/fklearn/training/transformation.py b/src/fklearn/training/transformation.py index a392d929..7243c57a 100644 --- a/src/fklearn/training/transformation.py +++ b/src/fklearn/training/transformation.py @@ -1027,7 +1027,7 @@ def missing_warner(df: pd.DataFrame, cols_list: List[str], cols_without_missing = df_selected.loc[:, df_selected.isna().sum(axis=0) == 0].columns.tolist() def p(dataset: pd.DataFrame) -> pd.DataFrame: - def detailed_assignment(df: pd.DataFrame, cols_to_check: List[str]) -> np.array: + def detailed_assignment(df: pd.DataFrame, cols_to_check: List[str]) -> np.ndarray: cols_with_missing = np.array([np.where(df[col].isna(), col, "") for col in cols_to_check]).T missing_by_row_list = np.array([list(filter(None, x)) for x in cols_with_missing]).reshape(-1, 1) if missing_by_row_list.size == 0: From 4a4435ad076ca5a5b670abfda3d3c24a617011c8 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 17:04:18 +0100 Subject: [PATCH 21/35] Change iterable type hint to iterator --- src/fklearn/validation/splitters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fklearn/validation/splitters.py b/src/fklearn/validation/splitters.py index 597f0173..1158c28c 100644 --- a/src/fklearn/validation/splitters.py +++ b/src/fklearn/validation/splitters.py @@ -1,7 +1,7 @@ import operator from datetime import datetime, timedelta from itertools import chain, repeat, starmap -from typing import Callable, Iterable, List, Tuple, Union +from typing import Callable, Iterator, List, Tuple, Union import numpy as np import pandas as pd @@ -383,7 +383,7 @@ def spatial_learning_curve_splitter(train_data: pd.DataFrame, time_column: str, training_limit: DateType, holdout_gap: timedelta = timedelta(days=0), - train_percentages: Iterable[float] = (0.25, 0.5, 0.75, 1.0), + train_percentages: Iterator[float] = (0.25, 0.5, 0.75, 1.0), random_state: int = None) -> SplitterReturnType: """ Splits the data for a spatial learning curve. Progressively adds more and From a21bb00b75b6db81bbfff404a0ee31259380ade4 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 17:22:22 +0100 Subject: [PATCH 22/35] wrap iterator with array --- src/fklearn/validation/splitters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fklearn/validation/splitters.py b/src/fklearn/validation/splitters.py index 1158c28c..34f1c1c5 100644 --- a/src/fklearn/validation/splitters.py +++ b/src/fklearn/validation/splitters.py @@ -1,7 +1,7 @@ import operator from datetime import datetime, timedelta from itertools import chain, repeat, starmap -from typing import Callable, Iterator, List, Tuple, Union +from typing import Callable, Iterable, List, Tuple, Union import numpy as np import pandas as pd @@ -383,7 +383,7 @@ def spatial_learning_curve_splitter(train_data: pd.DataFrame, time_column: str, training_limit: DateType, holdout_gap: timedelta = timedelta(days=0), - train_percentages: Iterator[float] = (0.25, 0.5, 0.75, 1.0), + train_percentages: Iterable[float] = (0.25, 0.5, 0.75, 1.0), random_state: int = None) -> SplitterReturnType: """ Splits the data for a spatial learning curve. Progressively adds more and @@ -422,7 +422,7 @@ def spatial_learning_curve_splitter(train_data: pd.DataFrame, random_state : int A seed for the random number generator that shuffles the IDs. """ - if np.min(train_percentages) < 0 or np.max(train_percentages) > 1: + if np.min(np.array(train_percentages)) < 0 or np.max(np.array(train_percentages)) > 1: raise ValueError('Train percentages must be between 0 and 1') if isinstance(training_limit, str): From 120ebc58f0ddfaa165343c3b5935ec83dba19e46 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 18:04:58 +0100 Subject: [PATCH 23/35] add build docs step --- .github/workflows/{pr.yaml => push.yaml} | 46 ++++++++++++++++++++++-- .readthedocs.yml | 2 +- 2 files changed, 45 insertions(+), 3 deletions(-) rename .github/workflows/{pr.yaml => push.yaml} (61%) diff --git a/.github/workflows/pr.yaml b/.github/workflows/push.yaml similarity index 61% rename from .github/workflows/pr.yaml rename to .github/workflows/push.yaml index 6c2ef0b2..79c1b075 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/push.yaml @@ -1,4 +1,4 @@ -name: PR check suite +name: Push Suite on: [push] @@ -14,6 +14,7 @@ jobs: python-version: 3.9 - name: run: | + #!/bin/bash set -euxo pipefail python3 -m pip install -q flake8==3.8.4 python3 -m flake8 \ @@ -26,7 +27,6 @@ jobs: src/ tests/ test-suite: - runs-on: ubuntu-20.04 strategy: matrix: @@ -40,6 +40,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Set up venv run: | + #!/bin/bash set -euxo pipefail python3 -m venv env @@ -51,25 +52,66 @@ jobs: echo "pip version: $(python3 -m pip --version)" - name: Install dependencies run: | + #!/bin/bash set -euxo pipefail source env/bin/activate python3 -m pip install -e .[devel] - name: Pip check run: | + #!/bin/bash set -euxo pipefail source env/bin/activate pip check - name: Type check run: | + #!/bin/bash set -euxo pipefail source env/bin/activate python3 -m mypy src tests --config mypy.ini - name: Test with pytest run: | + #!/bin/bash set -euxo pipefail source env/bin/activate python3 -m pytest --cov-fail-under=95 --cov=fklearn tests/ + + build-docs: + requires: [linter, test-suite] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Set up venv + run: | + #!/bin/bash + set -euxo pipefail + + python3 -m venv env + source env/bin/activate + python3 -m pip install --upgrade pip setuptools + + echo "Python path: $(which python3)" + echo "Python version: $(python3 --version)" + echo "pip version: $(python3 -m pip --version)" + - name: Install dependencies + run: | + #!/bin/bash + set -euxo pipefail + source env/bin/activate + + python3 -m pip install -e .[devel] + python3 -m pip install -r docs/requirements.txt + - name: Generate docs + run: | + #!/bin/bash + set -euxo pipefail + source env/bin/activate + + cd docs/ && make html diff --git a/.readthedocs.yml b/.readthedocs.yml index fb854570..b87dd585 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -18,7 +18,7 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: 3.6 + version: 3.9 install: - requirements: docs/requirements.txt - method: pip From 1c62027e30cd4aa84d36aebada1607f6d7abd5f6 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 18:09:56 +0100 Subject: [PATCH 24/35] workflow syntax --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 79c1b075..0126eeb3 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -80,7 +80,7 @@ jobs: python3 -m pytest --cov-fail-under=95 --cov=fklearn tests/ build-docs: - requires: [linter, test-suite] + needs: [linter, test-suite] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 From 5c647c828f60d65f34f99b4c82484e54f384b931 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Thu, 23 Mar 2023 18:29:36 +0100 Subject: [PATCH 25/35] try fix build docs --- .github/workflows/push.yaml | 1 + docs/requirements.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 0126eeb3..d06b54d2 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -114,4 +114,5 @@ jobs: set -euxo pipefail source env/bin/activate + sudo apt-get install pandoc cd docs/ && make html diff --git a/docs/requirements.txt b/docs/requirements.txt index a3713841..a0d33cbb 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,5 @@ nbsphinx>=0.4.2,<1 Sphinx>=1.7.1,<2 sphinx-rtd-theme>=0.4.3,<1 +jinja2<3 +markupsafe==2.0.1 From bd99f816a1e5ebf7103587dcaabb0c6e3f361579 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Fri, 24 Mar 2023 12:54:51 +0100 Subject: [PATCH 26/35] test packinging in push; publish workflow; cleanup repo --- .circleci/config.yml | 209 --------------------- .github/workflows/github-actions-demo.yaml | 18 -- .github/workflows/publish.yaml | 29 +++ .github/workflows/push.yaml | 26 ++- scripts/build.sh | 12 -- scripts/build_docs.sh | 12 -- scripts/create_venv.sh | 13 -- scripts/helpers.sh | 12 -- scripts/init_pypirc.sh | 7 - scripts/lint.sh | 18 -- scripts/release.sh | 11 -- scripts/run_tests.sh | 12 -- scripts/run_type_check.sh | 10 - 13 files changed, 51 insertions(+), 338 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/workflows/github-actions-demo.yaml create mode 100644 .github/workflows/publish.yaml delete mode 100644 scripts/build.sh delete mode 100644 scripts/build_docs.sh delete mode 100644 scripts/create_venv.sh delete mode 100644 scripts/helpers.sh delete mode 100644 scripts/init_pypirc.sh delete mode 100755 scripts/lint.sh delete mode 100644 scripts/release.sh delete mode 100644 scripts/run_tests.sh delete mode 100755 scripts/run_type_check.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0a7c51ae..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,209 +0,0 @@ -.common-values: - - docker-image: &docker-image circleci/python:3.6 - - docker-image-python39: &docker-image-python39 circleci/python:3.9 - - working_directory: &working-directory ~/repo - - # Download and cache dependencies - restore-cache: &restore-cache - keys: - - v1-dependencies-{{ checksum "requirements.txt" }} - - create-venv: &create-venv - name: Create virtualenv - command: . scripts/create_venv.sh - - save-cache: &save-cache - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} - - install-package: &install-package - name: Install package - command: | - . scripts/helpers.sh - install_package [devel] - - install-pandoc: &install-pandoc - name: Install pandoc - command: sudo apt-get install pandoc - -.master-branch-filter: &master-branch-filter - filters: - branches: - ignore: master - -.requires-lint: &requires-lint - requires: - - lint - <<: *master-branch-filter - -.tags-only-filter: &tags-only-filter - filters: - branches: - ignore: /.*/ - tags: - only: /.*/ - -.requires-build: &requires-build - requires: - - build - <<: *tags-only-filter - -.lint-steps: &lint-steps - docker: - - image: *docker-image - working_directory: *working-directory - steps: - - checkout - - restore_cache: *restore-cache - - run: *create-venv - - run: - name: Check code style (PEP8) - command: . scripts/lint.sh - - save_cache: *save-cache - -.test-steps: &test-steps - docker: - - image: *docker-image - working_directory: *working-directory - steps: - - checkout - - restore_cache: *restore-cache - - run: *create-venv - - save_cache: *save-cache - - run: *install-package - - run: - name: Run tests - command: . scripts/run_tests.sh - -.test-steps-python39: &test-steps-python39 - << : *test-steps - docker: - - image: *docker-image-python39 - -.pip-check: &pip-check - docker: - - image: *docker-image - working_directory: *working-directory - steps: - - checkout - - restore_cache: *restore-cache - - run: *create-venv - - save_cache: *save-cache - - run: *install-package - - run: - name: Run pip check - command: | - . scripts/helpers.sh - activate_venv - pip check - -.build: &build - docker: - - image: *docker-image - working_directory: *working-directory - steps: - - checkout - - restore_cache: *restore-cache - - run: *create-venv - - save_cache: *save-cache - - run: - name: Build package - command: . scripts/build.sh - - persist_to_workspace: - root: ~/repo - paths: - - dist - -.build-docs: &build-docs - docker: - - image: *docker-image - working_directory: *working-directory - steps: - - checkout - - restore_cache: *restore-cache - - run: *create-venv - - save_cache: *save-cache - - run: *install-pandoc - - run: *install-package - - run: - name: Build documentation - command: . scripts/build_docs.sh - -.release: &release - docker: - - image: *docker-image - working_directory: *working-directory - steps: - - attach_workspace: - at: /tmp/package - - checkout - - restore_cache: *restore-cache - - run: *create-venv - - save_cache: *save-cache - - run: - name: init .pypirc - command: . scripts/init_pypirc.sh - - run: - name: Release package - command: . scripts/release.sh /tmp/package/dist - -.type-check: &type-check - docker: - - image: *docker-image - working_directory: *working-directory - steps: - - checkout - - restore_cache: *restore-cache - - run: *create-venv - - save_cache: *save-cache - - run: *install-package - - run: - name: Run type tests - command: . scripts/run_type_check.sh - -version: 2 -jobs: - lint: - <<: *lint-steps - test: - <<: *test-steps - test-python39: - <<: *test-steps-python39 - pip-check: - <<: *pip-check - build: - <<: *build - build-docs: - <<: *build-docs - release: - <<: *release - type-check: - <<: *type-check - - -workflows: - version: 2 - test-all: - jobs: - - lint: - <<: *master-branch-filter - - test: - <<: *requires-lint - - test-python39: - <<: *requires-lint - - pip-check: - <<: *requires-lint - - type-check: - <<: *requires-lint - - build-docs: - <<: *requires-lint - build-and-release: - jobs: - - build: - <<: *tags-only-filter - - release: - <<: *requires-build diff --git a/.github/workflows/github-actions-demo.yaml b/.github/workflows/github-actions-demo.yaml deleted file mode 100644 index e007988b..00000000 --- a/.github/workflows/github-actions-demo.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: GitHub Actions Demo -run-name: ${{ github.actor }} is testing out GitHub Actions 🚀 -on: [push] -jobs: - Explore-GitHub-Actions: - runs-on: ubuntu-latest - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - name: Check out repository code - uses: actions/checkout@v3 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: echo "🖥️ The workflow is now ready to test your code on the runner." - - name: List files in the repository - run: | - ls ${{ github.workspace }} - - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..bdec0b87 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,29 @@ +# https://github.com/actions/starter-workflows/blob/main/ci/python-publish.yml +name: Publish Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip install build + - name: Build package + run: python3 -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index d06b54d2..f3f45c03 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -1,9 +1,11 @@ -name: Push Suite +name: Test Suite -on: [push] +on: + push: + branches: + - * jobs: - linter: runs-on: ubuntu-20.04 steps: @@ -26,12 +28,28 @@ jobs: --max-line-length=120 \ src/ tests/ + test-packaging: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Build package + run: | + #!/bin/bash + set -euxo pipefail + + python3 -m pip install --upgrade pip + pip install build + python3 -m build + test-suite: runs-on: ubuntu-20.04 strategy: matrix: python-version: ["3.6", "3.7", "3.8", "3.9"] - steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100644 index 365d8228..00000000 --- a/scripts/build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!bin/bash - -set -e - -cur_dir=$(dirname ${BASH_SOURCE[0]}) -source $cur_dir/helpers.sh - -activate_venv - -venv/bin/python3 -m pip install wheel -venv/bin/python3 setup.py sdist -venv/bin/python3 setup.py bdist_wheel diff --git a/scripts/build_docs.sh b/scripts/build_docs.sh deleted file mode 100644 index 7678ed21..00000000 --- a/scripts/build_docs.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!bin/bash - -set -e - -cur_dir=$(dirname ${BASH_SOURCE[0]}) -source $cur_dir/helpers.sh - -activate_venv - -venv/bin/python3 -m pip install -r docs/requirements.txt - -cd docs/ && make html diff --git a/scripts/create_venv.sh b/scripts/create_venv.sh deleted file mode 100644 index 9260f299..00000000 --- a/scripts/create_venv.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -e - -cur_dir=$(dirname ${BASH_SOURCE[0]}) -source $cur_dir/helpers.sh - -/usr/local/bin/python3 -m venv venv - -activate_venv - -venv/bin/python3 -m pip install -q --upgrade setuptools -venv/bin/python3 -m pip install -q --upgrade pip diff --git a/scripts/helpers.sh b/scripts/helpers.sh deleted file mode 100644 index 903d12ae..00000000 --- a/scripts/helpers.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!bin/bash - -set -e - -activate_venv() { - . venv/bin/activate -} - -install_package() { - activate_venv - venv/bin/python3 -m pip install -e .$1 -} diff --git a/scripts/init_pypirc.sh b/scripts/init_pypirc.sh deleted file mode 100644 index e29ec7bc..00000000 --- a/scripts/init_pypirc.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!bin/bash - -set -e - -echo -e "[pypi]" >>~/.pypirc -echo -e "username = $PYPI_USER" >>~/.pypirc -echo -e "password = $PYPI_PASSWORD" >>~/.pypirc diff --git a/scripts/lint.sh b/scripts/lint.sh deleted file mode 100755 index 6592081b..00000000 --- a/scripts/lint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -cur_dir=$(dirname ${BASH_SOURCE[0]}) -source $cur_dir/helpers.sh - -activate_venv - -venv/bin/python3 -m pip install -q flake8==3.8.4 -venv/bin/python3 -m flake8 \ - --ignore=E731,W503 \ - --filename=*.py \ - --exclude=__init__.py \ - --show-source \ - --statistics \ - --max-line-length=120 \ - src/ tests/ diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100644 index 5791ecee..00000000 --- a/scripts/release.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!bin/bash - -set -e - -cur_dir=$(dirname ${BASH_SOURCE[0]}) -source $cur_dir/helpers.sh - -activate_venv - -venv/bin/python3 -m pip install -q twine -twine upload $1/* diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh deleted file mode 100644 index 89a0e0da..00000000 --- a/scripts/run_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!bin/bash - -set -e - -cur_dir=$(dirname ${BASH_SOURCE[0]}) -source $cur_dir/helpers.sh - -activate_venv - -venv/bin/python3 -m pytest --cov=fklearn tests/ - -codecov diff --git a/scripts/run_type_check.sh b/scripts/run_type_check.sh deleted file mode 100755 index 4257d4c0..00000000 --- a/scripts/run_type_check.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!bin/bash - -set -e - -cur_dir=$(dirname ${BASH_SOURCE[0]}) -source $cur_dir/helpers.sh - -activate_venv - -venv/bin/python3 -m mypy src tests --config mypy.ini From 725a8c1a3aaaf5b225b9926420cf95bdbf1c0aed Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Fri, 24 Mar 2023 12:57:29 +0100 Subject: [PATCH 27/35] push branch syntax --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index f3f45c03..40266bf1 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -3,7 +3,7 @@ name: Test Suite on: push: branches: - - * + - '*' jobs: linter: From a2178bf8265227e23744cc7c71eb9d91b4b3c728 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Fri, 24 Mar 2023 13:01:53 +0100 Subject: [PATCH 28/35] run for all branches --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 40266bf1..8a2e47a0 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -3,7 +3,7 @@ name: Test Suite on: push: branches: - - '*' + - '**' jobs: linter: From 913a9a8c371aaef0fed3de49897fbcb09fef1572 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Fri, 24 Mar 2023 13:10:12 +0100 Subject: [PATCH 29/35] rollback changes to coveragerc; increment setup classifiers --- .coveragerc | 5 +++++ setup.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index c7edcbb3..35dd4024 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,8 @@ +[run] +parallel = True +concurrency = multiprocessing +branch = True + [report] exclude_lines = raise * diff --git a/setup.py b/setup.py index 7698e48a..d494946d 100644 --- a/setup.py +++ b/setup.py @@ -46,5 +46,9 @@ def requirements_from_pip(filename='requirements.txt'): "all": all_deps}, include_package_data=True, zip_safe=False, - classifiers=['Programming Language :: Python :: 3.6']) - + classifiers=[ + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9' + ]) From c47b653f608ff1a7768e65e7009c22a03c8302df Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Fri, 24 Mar 2023 13:14:09 +0100 Subject: [PATCH 30/35] update coverage threshold --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 8a2e47a0..415a59c4 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -95,7 +95,7 @@ jobs: set -euxo pipefail source env/bin/activate - python3 -m pytest --cov-fail-under=95 --cov=fklearn tests/ + python3 -m pytest --cov-fail-under=94 --cov=fklearn tests/ build-docs: needs: [linter, test-suite] From f1314469f0495ba176cfd027dada0eab474398b0 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 27 Mar 2023 16:00:24 +0200 Subject: [PATCH 31/35] Review suggestions --- .github/workflows/publish.yaml | 2 +- .github/workflows/push.yaml | 6 +++--- docs/source/contributing.rst | 22 ++++++++++++++-------- setup.py | 2 +- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index bdec0b87..514c2e69 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - pip install build + python3 -m pip install build - name: Build package run: python3 -m build - name: Publish package diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 415a59c4..f09a2dcc 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -18,7 +18,7 @@ jobs: run: | #!/bin/bash set -euxo pipefail - python3 -m pip install -q flake8==3.8.4 + python3 -m pip install -q flake8 python3 -m flake8 \ --ignore=E731,W503 \ --filename=*.py \ @@ -42,7 +42,7 @@ jobs: set -euxo pipefail python3 -m pip install --upgrade pip - pip install build + python3 -m pip install build python3 -m build test-suite: @@ -81,7 +81,7 @@ jobs: set -euxo pipefail source env/bin/activate - pip check + python3 -m pip check - name: Type check run: | #!/bin/bash diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 14d00605..da2d5bde 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -55,7 +55,7 @@ Creating the virtual environment .. code-block:: bash # Use an ENV_DIR of you choice. We are using ~/venvs - python3.6 -m venv ~/venvs/fklearn-dev + python3 -m venv ~/venvs/fklearn-dev source ~/venvs/fklearn-dev/activate Install the requirements @@ -65,7 +65,7 @@ This command will install all the test dependencies. To install the package you .. code-block:: bash - pip install -qe .[test_deps] + python3 -m pip install -qe .[devel] First testing ~~~~~~~~~~~~~ @@ -74,7 +74,7 @@ The following command should run all tests, if every test pass, you should be re .. code-block:: bash - python -m pytest tests/ + python3 -m pytest tests/ Creating a development branch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -106,15 +106,15 @@ In this session we'll guide you on how to contribute with the code. This is a gu Code standards -------------- -This project is compatible only with python3.6 and follows the `pep8 style `_ +This project is compatible only with python 3.6 to 3.9 and follows the `pep8 style `_ And we use this `import formatting `_ In order to check if your code is following our codestyle, you can run from the root directory of the repo the next commands: .. code-block:: bash - python -m pip install -q flake8 - python -m flake8 \ + python3 -m pip install -q flake8 + python3 -m flake8 \ --ignore=E731,W503 \ --filename=\*.py \ --exclude=__init__.py \ @@ -123,6 +123,12 @@ In order to check if your code is following our codestyle, you can run from the --max-line-length=120 \ src/ tests/ +We also use mypy for type checking, which you can run with: + +.. code-block:: bash + + python3 -m mypy src tests --config mypy.ini + Run tests --------- @@ -131,13 +137,13 @@ After you finish your feature development or bug fix, you should run your tests, .. code-block:: bash - python -m pytest tests/ + python3 -m pytest tests/ Or if you want to run only one test: .. code-block:: bash - python -m pytest tests/test-file-name.py::test_method_name + python3 -m pytest tests/test-file-name.py::test_method_name You must write tests for every feature **always**, you can look at the other tests to have a better idea how we implement them. diff --git a/setup.py b/setup.py index d494946d..d6044868 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def requirements_from_pip(filename='requirements.txt'): setup(name=MODULE_NAME, description="Functional machine learning", url='https://github.com/nubank/{:s}'.format(REPO_NAME), - python_requires='>=3.6.2', + python_requires='>=3.6.2,<3.10', author="Nubank", package_dir={'': 'src'}, packages=find_packages('src'), From fb1d8d782058442f6c667c8ceafddfffb5337ca2 Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 27 Mar 2023 16:21:36 +0200 Subject: [PATCH 32/35] lint --- src/fklearn/validation/splitters.py | 4 ++-- tests/training/test_regression.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/fklearn/validation/splitters.py b/src/fklearn/validation/splitters.py index 34f1c1c5..abd07118 100644 --- a/src/fklearn/validation/splitters.py +++ b/src/fklearn/validation/splitters.py @@ -444,8 +444,8 @@ def spatial_learning_curve_splitter(train_data: pd.DataFrame, spatial_ids, lambda ids: (np.array(train_percentages) * len(ids)).astype(int), # Get the corresponding indices for each % lambda idx: np.split(spatial_ids, idx)[:-1], # Split spatial ids by the indices - lambda l: map(lambda x: x.tolist(), l), # Transform sub-arrays into sub-lists - lambda l: filter(None, l), # Drop empty sub-lists + lambda to_list: map(lambda x: x.tolist(), to_list), # Transform sub-arrays into sub-lists + lambda drop_empty: filter(None, drop_empty), # Drop empty sub-lists accumulate(operator.add) # Cumulative sum of lists ) diff --git a/tests/training/test_regression.py b/tests/training/test_regression.py index 3a6d241d..3b471c0c 100644 --- a/tests/training/test_regression.py +++ b/tests/training/test_regression.py @@ -301,12 +301,12 @@ def test_custom_supervised_model_learner(): expected_col_train = df_train_classification.columns.tolist() + ["prediction_0", "prediction_1"] expected_col_test = df_test_classification.columns.tolist() + ["prediction_0", "prediction_1"] - assert(Counter(expected_col_train) == Counter(pred_train_classification.columns.tolist())) - assert(Counter(expected_col_test) == Counter(pred_test_classification.columns.tolist())) - assert(pred_test_classification.prediction_0.max() <= 1) - assert(pred_test_classification.prediction_0.min() >= 0) - assert(pred_test_classification.prediction_1.max() <= 1) - assert(pred_test_classification.prediction_1.min() >= 0) + assert (Counter(expected_col_train) == Counter(pred_train_classification.columns.tolist())) + assert (Counter(expected_col_test) == Counter(pred_test_classification.columns.tolist())) + assert (pred_test_classification.prediction_0.max() <= 1) + assert (pred_test_classification.prediction_0.min() >= 0) + assert (pred_test_classification.prediction_1.max() <= 1) + assert (pred_test_classification.prediction_1.min() >= 0) custom_regression_learner = custom_supervised_model_learner( features=features, @@ -319,7 +319,7 @@ def test_custom_supervised_model_learner(): expected_col_train = df_train_regression.columns.tolist() + ["prediction"] expected_col_test = df_test_regression.columns.tolist() + ["prediction"] - assert(Counter(expected_col_train) == Counter(pred_train.columns.tolist())) - assert(Counter(expected_col_test) == Counter(pred_test.columns.tolist())) - assert((pred_test.columns == pred_train.columns).all()) - assert("prediction" in pred_test.columns) + assert (Counter(expected_col_train) == Counter(pred_train.columns.tolist())) + assert (Counter(expected_col_test) == Counter(pred_test.columns.tolist())) + assert ((pred_test.columns == pred_train.columns).all()) + assert ("prediction" in pred_test.columns) From 1f18eb5d2cb0587e8bf7b9df5a583dd51adfbe2c Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 27 Mar 2023 16:42:15 +0200 Subject: [PATCH 33/35] Set version bump --- CHANGELOG.md | 5 +++++ src/fklearn/resources/VERSION | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f428146e..d046d38e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [2.2.2] - 2023-03-27 +- **Enhacement** + - Bump maximum allowed `scikit-learn` + - Move from CircleCI to Github Actions + ## [2.2.1] - 2022-09-06 - **Bug Fix** - Including a necessary init file to allow the import of the causal cate learners. diff --git a/src/fklearn/resources/VERSION b/src/fklearn/resources/VERSION index c043eea7..b1b25a5f 100644 --- a/src/fklearn/resources/VERSION +++ b/src/fklearn/resources/VERSION @@ -1 +1 @@ -2.2.1 +2.2.2 From dbadbec32ba91a6dc2f9bce990b136aa7ac64f9d Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 27 Mar 2023 17:00:21 +0200 Subject: [PATCH 34/35] Add previous changes to changelog --- CHANGELOG.md | 5 ++++- src/fklearn/resources/VERSION | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d046d38e..4a2a0240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,12 @@ # Changelog -## [2.2.2] - 2023-03-27 +## [2.3.0] - 2023-03-27 - **Enhacement** - Bump maximum allowed `scikit-learn` - Move from CircleCI to Github Actions + - Add optional `weight_column` argument for evaluators + - Change default of `min_df` from 20 to 1 on `TfidfVectorizer` + - Include new optional LGBM parameters to `lgbm_classification_learner` ## [2.2.1] - 2022-09-06 - **Bug Fix** diff --git a/src/fklearn/resources/VERSION b/src/fklearn/resources/VERSION index b1b25a5f..276cbf9e 100644 --- a/src/fklearn/resources/VERSION +++ b/src/fklearn/resources/VERSION @@ -1 +1 @@ -2.2.2 +2.3.0 From c1e87e78b4b6798cd0986c3ef3dd013074bd252f Mon Sep 17 00:00:00 2001 From: Flavio Pingas Date: Mon, 27 Mar 2023 17:07:49 +0200 Subject: [PATCH 35/35] change to rc --- CHANGELOG.md | 2 +- src/fklearn/resources/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a2a0240..f8dedcde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [2.3.0] - 2023-03-27 +## [2.3.0rc0] - 2023-03-27 - **Enhacement** - Bump maximum allowed `scikit-learn` - Move from CircleCI to Github Actions diff --git a/src/fklearn/resources/VERSION b/src/fklearn/resources/VERSION index 276cbf9e..bce4c60e 100644 --- a/src/fklearn/resources/VERSION +++ b/src/fklearn/resources/VERSION @@ -1 +1 @@ -2.3.0 +2.3.0rc0