diff --git a/.github/workflows/black_checker.yml b/.github/workflows/black_checker.yml
index c64d666027..fac1723682 100644
--- a/.github/workflows/black_checker.yml
+++ b/.github/workflows/black_checker.yml
@@ -1,6 +1,20 @@
 name: black-format-check
 
-on: [push, pull_request]
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 env:
   #If STRICT is set to true, it will fail on black check fail
diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml
index 376b628018..29eb0850dc 100644
--- a/.github/workflows/dist.yml
+++ b/.github/workflows/dist.yml
@@ -1,6 +1,20 @@
 name: dist-check
 
-on: [push, pull_request]
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 jobs:
   dist:
@@ -36,5 +50,9 @@ jobs:
     - name: PEP 561 Compliance
       run: |
         pip install mypy
+
         cd ..  # required to use the installed version of autosklearn
-        if ! python -c "import autosklearn"; then exit 1; fi
+
+        # Note this doesnt perform mypy checks, only
+        # that the types are exported
+        if ! mypy -c "import autosklearn"; then exit 1; fi
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
index 435168c9a8..3a9af5bf94 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -1,14 +1,16 @@
 #https://help.github.com/en/actions/language-and-framework-guides/publishing-docker-images#publishing-images-to-github-packages
 name: Publish Docker image
+
 on:
+
   push:
-    # Push  to `master` or `development`
     branches:
       - master
       - development
       - docker_workflow
 
 jobs:
+
   push_to_registry:
     name: Push Docker image to GitHub Packages
     runs-on: ubuntu-latest
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 5ab2f0d2ac..3645596c7b 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -1,11 +1,29 @@
 name: Docs
-on: [pull_request, push]
+
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 jobs:
+
   build-and-deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+
+    - name: Checkout
+      uses: actions/checkout@v2
       with:
         submodules: recursive
 
diff --git a/.github/workflows/isort_checker.yml b/.github/workflows/isort_checker.yml
index eba534d428..4f1f03f5a8 100644
--- a/.github/workflows/isort_checker.yml
+++ b/.github/workflows/isort_checker.yml
@@ -1,6 +1,20 @@
 name: isort-check
 
-on: [push, pull_request, workflow_dispatch]
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 env:
   #If STRICT is set to true, it will fail on isort check fail
@@ -8,7 +22,7 @@ env:
 
 jobs:
 
-  black-format-check:
+  isort-format-check:
     runs-on: ubuntu-latest
     steps:
 
diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
index da8d56db46..03ca861dff 100644
--- a/.github/workflows/pre-commit.yaml
+++ b/.github/workflows/pre-commit.yaml
@@ -1,6 +1,20 @@
 name: pre-commit
 
-on: [push, pull_request]
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 jobs:
   run-all-files:
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 5dbe04d404..4a9feba75f 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -1,106 +1,138 @@
 name: Tests
 
 on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
   push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
   pull_request:
-  workflow_dispatch:
+    branches:
+      - master
+      - development
+
   schedule:
-    # Every Monday at 7AM UTC
-    - cron: '0 07 * * 1'
+    # Every day at 7AM UTC
+    - cron: '0 07 * * *'
+
+env:
+
+  # Arguments used for pytest
+  pytest-args: >-
+    --forked
+    --durations=20
+    --timeout=300
+    --timeout-method=thread
+    -s
+
+  # Arguments used for code-cov which is later used to annotate PR's on github
+  code-cov-args: >-
+    --cov=autosklearn
+    --cov-report=xml
 
 jobs:
+
   ubuntu:
-    runs-on: ubuntu-20.04
+
+    name: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}
+    runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
+        os: [windows-latest, macos-latest, ubuntu-latest]
         python-version: ['3.7', '3.8', '3.9', '3.10']
-        use-conda: [true, false]
-        use-dist: [false]
+        kind: ['conda', 'source', 'dist']
+
+        exclude:
+          # Exclude all configurations *-*-dist, include one later
+          - kind: 'dist'
+
+          # Exclude windows as bash commands wont work in windows runner
+          - os: windows-latest
+
+          # Exclude macos as there are permission errors using conda as we do
+          - os: macos-latest
+
         include:
-          - python-version: '3.8'
+          # Add the tag code-cov to ubuntu-3.7-source
+          - os: ubuntu-latest
+            python-version: 3.7
+            kind: 'source'
             code-cov: true
-          - python-version: '3.7'
-            use-conda: false
-            use-dist: true
-      fail-fast: false
+
+           # Include one config with dist, ubuntu-3.7-dist
+          - os: ubuntu-latest
+            python-version: 3.7
+            kind: 'dist'
 
     steps:
 
-    - uses: actions/checkout@v2
+    - name: Checkout
+      uses: actions/checkout@v2
       with:
         submodules: recursive
 
     - name: Setup Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
-      # A note on checkout: When checking out the repository that
-      # triggered a workflow, this defaults to the reference or SHA for that event.
-      # Otherwise, uses the default branch (master) is used.
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Conda Install test dependencies
-      if: matrix.use-conda == true
+    - name: Conda install
+      if: matrix.kind == 'conda'
       run: |
         # Miniconda is available in $CONDA env var
         $CONDA/bin/conda create -n testenv --yes pip wheel gxx_linux-64 gcc_linux-64 swig python=${{ matrix.python-version }}
         $CONDA/envs/testenv/bin/python3 -m pip install --upgrade pip
         $CONDA/envs/testenv/bin/pip3 install -e .[test]
 
-    - name: Install test dependencies
-      if: matrix.use-conda == false && matrix.use-dist == false
+    - name: Source install
+      if: matrix.kind == 'source'
       run: |
         python -m pip install --upgrade pip
-        if [[ `python -c 'import platform; print(platform.python_version())' | cut -d '.' -f 2` -eq 6 ]]; then
-          # Numpy 1.20 dropped suppert for Python3.6
-          pip install "numpy<=1.19"
-        fi
         pip install -e .[test]
-        sudo apt-get update
-        sudo apt-get remove swig
-        sudo apt-get install swig3.0
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
 
-    - name: Dist Install test dependencies
-      if: matrix.use-conda == false && matrix.use-dist == true
+    - name: Dist install
+      if: matrix.kind == 'dist'
       run: |
         python -m pip install --upgrade pip
-        sudo apt-get update
-        sudo apt-get remove swig
-        sudo apt-get install swig3.0
-        sudo ln -s /usr/bin/swig3.0 /usr/bin/swig
-        # We need to install for the dependencies, like pytest
         python setup.py sdist
         last_dist=$(ls -t dist/auto-sklearn-*.tar.gz | head -n 1)
         pip install $last_dist[test]
 
-    - name: Store repository status
+    - name: Store git status
       id: status-before
       run: |
         echo "::set-output name=BEFORE::$(git status --porcelain -b)"
 
-    - name: Conda Run tests
+    - name: Tests
       timeout-minutes: 60
-      if: matrix.use-conda == true
       run: |
         export OPENBLAS_NUM_THREADS=1
         export OMP_NUM_THREADS=1
         export MKL_NUM_THREADS=1
-        # We activate conda as metalearning uses python directly, so we need
-        # to change the default python
-        export PATH="$CONDA/envs/testenv/bin:$PATH"
-        if [ ${{ matrix.code-cov }} ]; then codecov='--cov=autosklearn --cov-report=xml'; fi
-        $CONDA/envs/testenv/bin/python3 -m pytest --durations=20 --timeout=300 --timeout-method=thread -v $codecov test
 
-    - name: Run tests
-      timeout-minutes: 60
-      if: matrix.use-conda == false
-      run: |
-        export OPENBLAS_NUM_THREADS=1
-        export OMP_NUM_THREADS=1
-        export MKL_NUM_THREADS=1
-        if [ ${{ matrix.code-cov }} ]; then codecov='--cov=autosklearn --cov-report=xml'; fi
-        pytest --durations=20 --timeout=300 --timeout-method=thread -v $codecov test
+        if [[ ${{ matrix.kind }} == 'conda' ]]; then
+          PYTHON=$CONDA/envs/testenv/bin/python3
+
+          # As one of the tests runs a subprocess command and calls `python3`, we must
+          # explicitly add it to the path
+          export PATH="$CONDA/envs/testenv/bin:$PATH"
+
+        else
+          PYTHON=$(which python3)
+        fi
+
+        if [ ${{ matrix.code-cov }} ]; then
+          $PYTHON -m pytest ${{ env.pytest-args }} ${{ env.code-cov-args }} test
+        else
+          $PYTHON -m pytest ${{ env.pytest-args }} test
+        fi
 
     - name: Check for files left behind by test
       if: ${{ always() }}
@@ -116,7 +148,7 @@ jobs:
 
     - name: Upload coverage
       if: matrix.code-cov && always()
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2
       with:
         fail_ci_if_error: true
         verbose: true
diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml
index 45422d04eb..d95d344674 100644
--- a/.github/workflows/stale.yaml
+++ b/.github/workflows/stale.yaml
@@ -1,9 +1,11 @@
 name: 'Close stale issues'
+
 on:
   schedule:
-    - cron: '30 1 * * *'
+    - cron: '0 7 * * *'
 
 jobs:
+
   stale:
     runs-on: ubuntu-latest
     steps:
@@ -11,12 +13,14 @@ jobs:
         with:
           days-before-stale: 60
           days-before-close: 7
+          stale-issue-label: 'stale'
+          only-issue-labels: 'Answered,Feedback-Required,invalid,wontfix'
+          exempt-all-milestones: true
+
           stale-issue-message: >
             This issue has been automatically marked as stale because it has not had
             recent activity. It will be closed if no further activity occurs for the
             next 7 days. Thank you for your contributions.
+
           close-issue-message: >
             This issue has been automatically closed due to inactivity.
-          stale-issue-label: 'stale'
-          only-issue-labels: 'Answered,Feedback-Required,invalid,wontfix'
-          exempt-all-milestones: true
diff --git a/MANIFEST.in b/MANIFEST.in
index 37e70af053..dffd0c7283 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,6 +2,7 @@ include LICENSE.txt
 include requirements.txt
 include autosklearn/util/logging.yaml
 include autosklearn/requirements.txt
+include autosklearn/py.typed
 
 # Meta-data
 recursive-include autosklearn/metalearning/files *.arff *.csv *.txt
diff --git a/autosklearn/py.typed b/autosklearn/py.typed
new file mode 100644
index 0000000000..e69de29bb2