Merge pull request #455 from quantumblacklabs/release/0.15.6

Release/0.15.6
kedro-org · Feb 26, 2020 · eb6bdd8 · eb6bdd8
2 parents 98a6c8f + 31e281e
commit eb6bdd8
Show file tree

Hide file tree

Showing 350 changed files with 15,589 additions and 1,452 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -36,7 +36,6 @@ utils:
       pre-commit install --install-hooks
       pre-commit install --hook-type pre-push
 
-
   linters: &linters
     name: Run pylint and flake8
     command: |
@@ -52,8 +51,7 @@ utils:
     command: |
       make e2e-tests
 
-  build_docs: &build_docs
-    # NOTE: doesn't work on python 3.5
+  build_docs: &build_docs # NOTE: doesn't work on python 3.5
     name: Build documentation
     command: |
       make build-docs
@@ -63,6 +61,11 @@ utils:
     command: |
       make legal
 
+  pip_compile: &pip_compile
+    name: pip-compile requirements file
+    command: |
+      make pip-compile
+
   steps_unit_tests: &steps_unit_tests
     steps:
       - checkout
@@ -86,14 +89,20 @@ utils:
       - run: *setup_requirements
       - run: *e2e_tests
 
-  steps_build_docs: &steps_build_docs
-    # NOTE: requires python3.6+
+  steps_build_docs: &steps_build_docs # NOTE: requires python3.6+
     steps:
       - checkout
       - run: *setup_conda
       - run: *setup_requirements
       - run: *build_docs
 
+  steps_pip_compile: &steps_pip_compile
+    steps:
+      - checkout
+      - run: *setup_conda
+      - run: *setup_requirements
+      - run: *pip_compile
+
 jobs:
   unit_tests_35:
     docker:
@@ -150,9 +159,14 @@ jobs:
       - image: *py37
     <<: *steps_build_docs
 
+  pip_compile:
+    docker:
+      - image: *py37
+    <<: *steps_pip_compile
+
   all_circleci_checks_succeeded:
     docker:
-      - image: circleci/python  # any light-weight image
+      - image: circleci/python # any light-weight image
 
     steps:
       - run:
@@ -174,6 +188,7 @@ workflows:
       - linters_37
       - e2e_tests_37
       - docs_37
+      - pip_compile
       - all_circleci_checks_succeeded:
           requires:
             - unit_tests_35
@@ -187,3 +202,4 @@ workflows:
             - linters_37
             - e2e_tests_37
             - docs_37
+            - pip_compile
diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md
@@ -0,0 +1,14 @@
+---
+name: Ask a question
+about: Let us help you with any queries you may have while using Kedro
+title: '<Question>'
+labels: 'Issue: Question'
+assignees: ''
+
+---
+
+## What are you trying to do?
+Tell us what you are trying to accomplish and any methods you may have tried.
+
+## Preferred channel
+We will do our best to help you with your query but in future, head to Stack Overflow for questions like this. Tag your questions with [`kedro`](https://stackoverflow.com/questions/tagged/kedro) and we'll get a notification.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,16 +1,8 @@
-## Notice
-
-- [ ] I acknowledge and agree that, by checking this box and clicking "Submit Pull Request":
-
-- I submit this contribution under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0.txt) and represent that I am entitled to do so on behalf of myself, my employer, or relevant third parties, as applicable.
-- I certify that (a) this contribution is my original creation and / or (b) to the extent it is not my original creation, I am authorised to submit this contribution on behalf of the original creator(s) or their licensees.
-- I certify that the use of this contribution as authorised by the Apache 2.0 license does not violate the intellectual property rights of anyone else.
-
-## Motivation and Context
-Why was this PR created?
+## Description
+<!-- Why was this PR created? -->
 
-## How has this been tested?
-What testing strategies have you used?
+## Development notes
+<!-- What have you changed, and how has this been tested? -->
 
 ## Checklist
 
@@ -19,4 +11,11 @@ What testing strategies have you used?
 - [ ] Updated the documentation to reflect the code changes
 - [ ] Added a description of this change and added my name to the list of supporting contributions in the [`RELEASE.md`](/RELEASE.md) file
 - [ ] Added tests to cover my changes
-- [ ] Assigned myself to the PR
+
+## Notice
+
+- [ ] I acknowledge and agree that, by checking this box and clicking "Submit Pull Request":
+
+- I submit this contribution under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0.txt) and represent that I am entitled to do so on behalf of myself, my employer, or relevant third parties, as applicable.
+- I certify that (a) this contribution is my original creation and / or (b) to the extent it is not my original creation, I am authorised to submit this contribution on behalf of the original creator(s) or their licensees.
+- I certify that the use of this contribution as authorised by the Apache 2.0 license does not violate the intellectual property rights of anyone else.
diff --git a/.isort.cfg b/.isort.cfg
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,9 +8,7 @@ repos:
       rev: v2.2.3
       hooks:
           - id: trailing-whitespace
-            stages: [commit, manual]
           - id: end-of-file-fixer
-            stages: [commit, manual]
           - id: check-yaml # Checks yaml files for parseable syntax.
             exclude: "^kedro/template/"
           - id: check-json # Checks json files for parseable syntax.
@@ -19,17 +17,10 @@ repos:
           - id: check-merge-conflict # Check for files that contain merge conflict strings.
           - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source.
             exclude: "^kedro/template/"
-          - id: detect-private-key # Detects the presence of private keys
           - id: requirements-txt-fixer # Sorts entries in requirements.txt
           - id: flake8
             exclude: "^kedro/template/"
 
-    - repo: https://github.com/pre-commit/mirrors-isort
-      rev: v4.3.21
-      hooks:
-          - id: isort
-            exclude: "^kedro/template/"
-
     - repo: https://github.com/pre-commit/mirrors-mypy
       rev: v0.720
       hooks:
@@ -106,7 +97,7 @@ repos:
             name: "Black"
             language: system
             pass_filenames: false
-            entry: python -m tools.min_version 3.6 "pip install black" "black kedro extras features tests"
+            entry: python -m tools.min_version 3.6 "black kedro extras features tests"
           - id: legal
             name: "Licence check"
             language: system
@@ -116,4 +107,15 @@ repos:
             name: "Import Linter"
             language: system
             pass_filenames: false
-            entry: python -m tools.min_version 3.6 "pip install import-linter" lint-imports
+            entry: python -m tools.min_version 3.6 lint-imports
+          - id: secret_scan
+            name: "Secret scan"
+            language: system
+            pass_filenames: false
+            entry: make secret-scan
+          - id: isort
+            name: "Sort imports"
+            language: system
+            types: [file, python]
+            exclude: ^kedro/template/
+            entry: isort
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -32,12 +32,12 @@ If you have new ideas for Kedro functionality then please open a [GitHub issue](
 
 If you're unsure where to begin contributing to Kedro, please start by looking through the `good first issues` and `help wanted issues` on [GitHub](https://github.com/quantumblacklabs/kedro/issues).
 
-We focus on three areas for contribution: `core`, [`contrib`](/kedro/contrib/) or `plugin`:
+We focus on three areas for contribution: `core`, [`extras`](/kedro/extras/) or `plugin`:
 - `core` refers to the primary Kedro library
-- [`contrib`](/kedro/contrib/) refers to features that could be added to `core` that do not introduce too many depencies or require new Kedro CLI commands to be created e.g. adding a new dataset to the `io` data management module
-- [`plugin`](https://kedro.readthedocs.io/en/latest/04_user_guide/10_developing_plugins.html) refers to new functionality that requires a Kedro CLI command e.g. adding in Airflow functionality
+- [`extras`](/kedro/extras/) refers to features that could be added to `core` that do not introduce too many depencies or require new Kedro CLI commands to be created e.g. adding a new dataset to the `kedro.extras.dataset` data management module. All the datasets are placed under `kedro.extras.datasets` to separate heavy depencies (e.g Pandas) from Kedro `core` components.
+- [`plugin`](https://kedro.readthedocs.io/en/stable/04_user_guide/10_developing_plugins.html) refers to new functionality that requires a Kedro CLI command e.g. adding in Airflow functionality
 
-Typically, we only accept small contributions for the `core` Kedro library but accept new features as `plugin`s or additions to the [`contrib`](/kedro/contrib/) module. We regularly review [`contrib`](/kedro/contrib/) and may migrate modules to `core` if they prove to be essential for the functioning of the framework or if we believe that they are used by most projects.
+Typically, we only accept small contributions for the `core` Kedro library but accept new features as `plugin`s or additions to the [`extras`](/kedro/extras/) module. We regularly review [`extras`](/kedro/extras/) and may migrate modules to `core` if they prove to be essential for the functioning of the framework.
 
 ## Your first contribution
 
@@ -71,13 +71,12 @@ There is an automated check to verify that it exists. The check will highlight a
 ### Branching conventions
 We use a branching model that helps us keep track of branches in a logical, consistent way. All branches should have the hyphen-separated convention of: `<type-of-change>/<short-description-of-change>` e.g. `contrib/io-dataset`
 
-| Types of changes | Description                                                                  |
-| ---------------- | ---------------------------------------------------------------------------- |
-| `contrib`        | Changes under `contrib/` and has no side-effects to other `contrib/` modules |
-| `docs`           | Changes to the documentation under `docs/source/`                            |
-| `feature`        | Non-breaking change which adds functionality                                 |
-| `fix`            | Non-breaking change which fixes an issue                                     |
-| `tests`          | Changes to project unit `tests/` and / or integration `features/` tests      |
+| Types of changes | Description                                                             |
+| ---------------- | ----------------------------------------------------------------------- |
+| `docs`           | Changes to the documentation under `docs/source/`                       |
+| `feature`        | Non-breaking change which adds functionality                            |
+| `fix`            | Non-breaking change which fixes an issue                                |
+| `tests`          | Changes to project unit `tests/` and / or integration `features/` tests |
 
 ## `core` contribution process
 
@@ -92,13 +91,13 @@ Small contributions are accepted for the `core` library:
 
  > _Note:_ We will work with you to complete your contribution but we reserve the right to takeover abandoned PRs.
 
-## `contrib` contribution process
+## `extras` contribution process
 
-You can add new work to `contrib` if you do not need to create a new Kedro CLI command:
+You can add new work to `extras` if you do not need to create a new Kedro CLI command:
 
  1. Create an [issue](https://github.com/quantumblacklabs/kedro/issues) describing your contribution.
  2. Fork the project by clicking **Fork** in the top-right corner of the [Kedro GitHub repository](https://github.com/quantumblacklabs/kedro) and then choosing the target account the repository will be forked to.
- 3. Work in [`contrib`](/kedro/contrib/) and create a feature branch on your forked repository and push all your local changes to that feature branch.
+ 3. Work in [`extras`](/kedro/extras/) and create a feature branch on your forked repository and push all your local changes to that feature branch.
  4. Before submitting a pull request, please ensure that unit, e2e tests and linting are passing for your changes by running `make test`, `make e2e-tests` and `make lint` locally, have a look at the section [Running checks locally](/CONTRIBUTING.md#running-checks-locally) below.
  5. Include a `README.md` with instructions on how to use your contribution.
  6. Open a PR against the `quantumblacklabs:develop` branch from your feature branch and reference your issue in the PR description (e.g., `Resolves #<issue-number>`).
@@ -109,7 +108,7 @@ You can add new work to `contrib` if you do not need to create a new Kedro CLI c
 
 ## `plugin` contribution process
 
-See the [`plugin` development documentation](https://kedro.readthedocs.io/en/latest/04_user_guide/10_developing_plugins.html) for guidance on how to design and develop a Kedro `plugin`.
+See the [`plugin` development documentation](https://kedro.readthedocs.io/en/stable/04_user_guide/10_developing_plugins.html) for guidance on how to design and develop a Kedro `plugin`.
 
 ## CI / CD and running checks locally
 To run E2E tests you need to install the test requirements which includes `behave`.
@@ -121,6 +120,8 @@ make install-test-requirements
 make install-pre-commit
 ```
 
+> _Note:_ If Spark/PySpark/Hive tests for datasets are failing it might be due to the lack of Java>8 support from Spark. You can try using `export JAVA_HOME=$(/usr/libexec/java_home -v 1.8)` which works under MacOS or other workarounds. [Reference](https://stackoverflow.com/questions/53583199/pyspark-error-unsupported-class-file-major-version-55)
+
 ### Running checks locally
 
 All checks run by our CI / CD servers can be run locally on your computer.

diff --git a/LICENSE.md b/LICENSE.md
@@ -1,4 +1,4 @@
-Copyright 2018-2019 QuantumBlack Visual Analytics Limited
+Copyright 2020 QuantumBlack Visual Analytics Limited
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

diff --git a/Makefile b/Makefile
@@ -24,6 +24,12 @@ test:
 e2e-tests:
 	behave
 
+pip-compile:
+	pip-compile -q -o -
+
+secret-scan:
+	trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt .
+
 SPHINXPROJ = Kedro
 
 build-docs: