Merge branch 'apache:master' into handle-time-column

.asf.yaml

@@ -66,12 +66,12 @@ github:
           - cypress-matrix (3, chrome)
           - docker-build
           - frontend-build
-          - pre-commit (3.7)
-          - python-lint (3.7)
-          - test-mysql (3.7)
-          - test-postgres (3.7)
+          - pre-commit (3.8)
+          - python-lint (3.8)
+          - test-mysql (3.8)
           - test-postgres (3.8)
-          - test-sqlite (3.7)
+          - test-postgres (3.9)
+          - test-sqlite (3.8)
 
       required_pull_request_reviews:
         dismiss_stale_reviews: false

.github/CODEOWNERS

@@ -6,3 +6,6 @@
 .github/workflows/ecs-task-definition.json @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 .github/workflows/docker-ephemeral-env.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 .github/workflows/ephemeral*.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
+
+# Notify some committers of changes in the Select component
+/superset-frontend/src/components/Select/ @michael-s-molina @geido

.github/workflows/superset-e2e.yml

@@ -67,7 +67,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.8"
       - name: OS dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

.github/workflows/superset-frontend.yml

@@ -18,6 +18,8 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check npm lock file version
+        run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
       - name: Check if frontend changes are present
         id: check
         env:
@@ -26,9 +28,10 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh frontend
       - name: Setup Node.js
+        if: steps.check.outcome == 'failure'
         uses: actions/setup-node@v2
         with:
-          node-version:  '16'
+          node-version: "16"
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

.github/workflows/superset-helm-lint.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.8
 
       - name: Set up chart-testing
         uses: ./.github/actions/chart-testing-action

.github/workflows/superset-python-integrationtest.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -77,7 +77,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8, 3.9]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -141,7 +141,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config

.github/workflows/superset-python-misc.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -29,6 +29,7 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
@@ -50,7 +51,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -76,7 +77,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2

.github/workflows/superset-python-unittest.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7,3.8]
+        python-version: [3.8, 3.9]
     env:
       PYTHONPATH: ${{ github.workspace }}
     steps:

.github/workflows/superset-translations.yml

@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -15,12 +15,8 @@
 # limitations under the License.
 #
 repos:
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v1.9.3
-    hooks:
-      - id: seed-isort-config
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.9.3
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
@@ -48,3 +44,8 @@ repos:
     hooks:
       - id: black
         language_version: python3
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.2.1 # Use the sha or tag you want to point at
+    hooks:
+      - id: prettier
+        files: 'superset-frontend'

CONTRIBUTING.md

@@ -650,23 +650,36 @@ pre-commit run --all-files
 
 ## Linting
 
-Lint the project with:
+### Python
+
+We use [Pylint](https://pylint.org/) for linting which can be invoked via:
 
 ```bash
 # for python
 tox -e pylint
+```
 
-Alternatively, you can use pre-commit (mentioned above) for python linting
+In terms of best practices please advoid blanket disablement of Pylint messages globally (via `.pylintrc`) or top-level within the file header, albeit there being a few exceptions. Disablement should occur inline as it prevents masking issues and provides context as to why said message is disabled.
 
-The Python code is auto-formatted using [Black](https://github.com/python/black) which
+Additionally the Python code is auto-formatted using [Black](https://github.com/python/black) which
 is configured as a pre-commit hook. There are also numerous [editor integrations](https://black.readthedocs.io/en/stable/editor_integration.html)
 
-# for frontend
+### TypeScript
+
+```bash
 cd superset-frontend
 npm ci
 npm run lint
 ```
 
+If using the eslint extension with vscode, put the following in your workspace `settings.json` file:
+
+```json
+"eslint.workingDirectories": [
+  "superset-frontend"
+]
+```
+
 ## Conventions
 
 ### Python
@@ -891,7 +904,7 @@ apt install -y net-tools
 pip install debugpy
 ```
 
-Find the PID for the Flask process. Make sure to use the first PID. The Flask app will re-spawn a sub-process everytime you change any of the python code. So it's important to use the first PID.
+Find the PID for the Flask process. Make sure to use the first PID. The Flask app will re-spawn a sub-process every time you change any of the python code. So it's important to use the first PID.
 
 ```bash
 ps -ef
@@ -962,7 +975,7 @@ Once the pod is running as root and has the SYS_PTRACE capability it will be abl
 
 You can follow the same instructions as in the docker-compose. Enter the pod and install the required library and packages; gdb, netstat and debugpy.
 
-Often in a kuernetes environment nodes are not addressable from ouside the cluster. VSCode will thus be unable to remotely connect to port 5678 on a kubernetes node. In order to do this you need to create a tunnel that port forwards 5678 to your local machine.
+Often in a Kubernetes environment nodes are not addressable from outside the cluster. VSCode will thus be unable to remotely connect to port 5678 on a Kubernetes node. In order to do this you need to create a tunnel that port forwards 5678 to your local machine.
 
 ```
 kubectl port-forward  pod/superset-<some random id> 5678:5678
@@ -1328,7 +1341,7 @@ Chart parameters are stored as a JSON encoded string the `slices.params` column
 
 The following tables provide a non-exhausive list of the various fields which can be present in the JSON object grouped by the Explorer pane sections. These values were obtained by extracting the distinct fields from a legacy deployment consisting of tens of thousands of charts and thus some fields may be missing whilst others may be deprecated.
 
-Note not all fields are correctly catagorized. The fields vary based on visualization type and may apprear in different sections depending on the type. Verified deprecated columns may indicate a missing migration and/or prior migrations which were unsucessful and thus future work may be required to clean up the form-data.
+Note not all fields are correctly categorized. The fields vary based on visualization type and may apprear in different sections depending on the type. Verified deprecated columns may indicate a missing migration and/or prior migrations which were unsuccessful and thus future work may be required to clean up the form-data.
 
 ### Datasource & Chart Type
 

Dockerfile

@@ -18,7 +18,7 @@
 ######################################################################
 # PY stage that simply does a pip install on our requirements
 ######################################################################
-ARG PY_VER=3.7.9
+ARG PY_VER=3.8.12
 FROM python:${PY_VER} AS superset-py
 
 RUN mkdir /app \
@@ -73,7 +73,7 @@ RUN cd /app/superset-frontend \
 ######################################################################
 # Final lean image...
 ######################################################################
-ARG PY_VER=3.7.9
+ARG PY_VER=3.8.12
 FROM python:${PY_VER} AS lean
 
 ENV LANG=C.UTF-8 \
@@ -94,7 +94,7 @@ RUN mkdir -p ${PYTHONPATH} \
             libpq-dev \
         && rm -rf /var/lib/apt/lists/*
 
-COPY --from=superset-py /usr/local/lib/python3.7/site-packages/ /usr/local/lib/python3.7/site-packages/
+COPY --from=superset-py /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/site-packages/
 # Copying site-packages doesn't move the CLIs, so let's copy them one by one
 COPY --from=superset-py /usr/local/bin/gunicorn /usr/local/bin/celery /usr/local/bin/flask /usr/bin/
 COPY --from=superset-node /app/superset/static/assets /app/superset/static/assets

Makefile

@@ -15,8 +15,8 @@
 # limitations under the License.
 #
 
-# Python version installed; we need 3.8 or 3.7
-PYTHON=`command -v python3.8 || command -v python3.7`
+# Python version installed; we need 3.7-3.9
+PYTHON=`command -v python3.9 || command -v python3.8 || command -v python3.7`
 
 .PHONY: install superset venv pre-commit
 
@@ -62,7 +62,7 @@ update-js:
 
 venv:
 	# Create a virtual environment and activate it (recommended)
-	if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.7 or 3.8 installed"; exit 1; fi
+	if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.7, 3.8 or 3.9 installed"; exit 1; fi
 	test -d venv || ${PYTHON} -m venv venv # setup a python3 virtualenv
 	. venv/bin/activate
 

RELEASING/Dockerfile.from_local_tarball

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 
 RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
 

RELEASING/Dockerfile.from_svn_tarball

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 
 RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
 

RELEASING/Dockerfile.make_docs

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 ARG VERSION
 
 RUN git clone --depth 1 --branch ${VERSION} https://github.com/apache/superset.git /superset

RELEASING/Dockerfile.make_tarball

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 
 RUN apt-get update -y
 RUN apt-get install -y jq

RELEASING/README.md

@@ -179,7 +179,7 @@ the tag and create a signed source tarball from it:
 Note that `make_tarball.sh`:
 
 - By default assumes you have already executed an SVN checkout to `$HOME/svn/superset_dev`.
-This can be overriden by setting `SUPERSET_SVN_DEV_PATH` environment var to a different svn dev directory
+This can be overridden by setting `SUPERSET_SVN_DEV_PATH` environment var to a different svn dev directory
 - Will refuse to craft a new release candidate if a release already exists on your local svn dev directory
 - Will check `package.json` version number and fails if it's not correctly set
 

RESOURCES/INTHEWILD.md

@@ -137,6 +137,9 @@ Join our growing community!
 - [TrustMedis](https://trustmedis.com) [@famasya]
 - [WeSure](https://www.wesure.cn/)
 
+### HR / Staffing
+- [Symmetrics](https://www.symmetrics.fyi)
+
 ### Others
  - [Grassroot](https://www.grassrootinstitute.org/)
  - [komoot](https://www.komoot.com/) [@christophlingg]

UPDATING.md

@@ -26,11 +26,14 @@ assists people when migrating to a new version.
 
 ### Breaking Changes
 
+- [17290](https://github.com/apache/superset/pull/17290): Bumps pandas to `1.3.4` and pyarrow to `5.0.0`
 - [16660](https://github.com/apache/incubator-superset/pull/16660): The `columns` Jinja parameter has been renamed `table_columns` to make the `columns` query object parameter available in the Jinja context.
 - [16711](https://github.com/apache/incubator-superset/pull/16711): The `url_param` Jinja function will now by default escape the result. For instance, the value `O'Brien` will now be changed to `O''Brien`. To disable this behavior, call `url_param` with `escape_result` set to `False`: `url_param("my_key", "my default", escape_result=False)`.
 
 ### Potential Downtime
 
+- [16756](https://github.com/apache/incubator-superset/pull/16756): a change which renames the `dbs.allow_csv_upload` column to `dbs.allow_file_upload` via a (potentially locking) DDL operation.
+
 ### Deprecations
 
 ### Other
@@ -46,7 +49,7 @@ assists people when migrating to a new version.
 
 ### Potential Downtime
 
-- [14234](https://github.com/apache/superset/pull/14234): Adds the `limiting_factor` column to the `query` table. Give the migration includes a DDL operation on a heavily trafficed table, potential service downtime may be required.
+- [14234](https://github.com/apache/superset/pull/14234): Adds the `limiting_factor` column to the `query` table. Give the migration includes a DDL operation on a heavily trafficked table, potential service downtime may be required.
 
 -[16454](https://github.com/apache/superset/pull/16454): Adds the `extra` column to the `table_columns` table. Users using MySQL will either need to schedule downtime or use the percona toolkit (or similar) to perform the migration.
 
@@ -229,7 +232,7 @@ assists people when migrating to a new version.
 - [8699](https://github.com/apache/superset/pull/8699): A `row_level_security_filters`
   table has been added, which is many-to-many with `tables` and `ab_roles`. The applicable filters
   are added to the sqla query, and the RLS ids are added to the query cache keys. If RLS is enabled in config.py (`ENABLE_ROW_LEVEL_SECURITY = True`; by default, it is disabled), they can be
-  accessed through the `Security` menu, or when editting a table.
+  accessed through the `Security` menu, or when editing a table.
 
 - [8732](https://github.com/apache/superset/pull/8732): Swagger user interface is now enabled by default.
   A new permission `show on SwaggerView` is created by `superset init` and given to the `Admin` Role. To disable the UI,
@@ -408,7 +411,7 @@ Superset 0.25.0 contains a backwards incompatible changes.
 If you run a production system you should schedule downtime for this
 upgrade.
 
-The PRs bellow have more information around the breaking changes:
+The PRs below have more information around the breaking changes:
 
 - [9825](https://github.com/apache/superset/pull/9825): Support for Excel sheet upload added. To enable support, install Superset with the optional dependency `excel`