Merge branch 'master' into elizabeth/update-mutator

.github/workflows/bashlib.sh

@@ -38,10 +38,10 @@ default-setup-command() {
 }
 
 apt-get-install() {
-    say "::group::apt-get install dependencies"
-    sudo apt-get update && sudo apt-get install --yes \
-      libsasl2-dev
-    say "::endgroup::"
+  say "::group::apt-get install dependencies"
+  sudo apt-get update && sudo apt-get install --yes \
+    libsasl2-dev
+  say "::endgroup::"
 }
 
 pip-upgrade() {
@@ -161,7 +161,7 @@ cypress-run() {
   if [[ -z $CYPRESS_KEY ]]; then
     $cypress --spec "cypress/integration/$page" --browser "$browser"
   else
-    export CYPRESS_RECORD_KEY=`echo $CYPRESS_KEY | base64 --decode`
+    export CYPRESS_RECORD_KEY=$(echo $CYPRESS_KEY | base64 --decode)
     # additional flags for Cypress dashboard recording
     $cypress --spec "cypress/integration/$page" --browser "$browser" \
       --record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \
@@ -190,8 +190,8 @@ cypress-run-all() {
   cat "$flasklog"
   say "::endgroup::"
 
-  # Rerun SQL Lab tests with backend persist enabled
-  export SUPERSET_CONFIG=tests.integration_tests.superset_test_config_sqllab_backend_persist
+  # Rerun SQL Lab tests with backend persist disabled
+  export SUPERSET_CONFIG=tests.integration_tests.superset_test_config_sqllab_backend_persist_off
 
   # Restart Flask with new configs
   kill $flaskProcessId

.github/workflows/embedded-sdk-test.yml

@@ -20,4 +20,5 @@ jobs:
           node-version: "16"
           registry-url: 'https://registry.npmjs.org'
       - run: npm ci
+      - run: npm test
       - run: npm run build

.github/workflows/superset-e2e.yml

@@ -24,7 +24,6 @@ jobs:
         browser: ["chrome"]
     env:
       FLASK_ENV: development
-      ENABLE_REACT_CRUD_VIEWS: true
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
       SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
       PYTHONPATH: ${{ github.workspace }}

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910
+    rev: v0.941
     hooks:
       - id: mypy
         additional_dependencies: [types-all]

CONTRIBUTING.md

@@ -198,7 +198,7 @@ Finally, never submit a PR that will put master branch in broken state. If the P
 #### Authoring
 
 - Fill in all sections of the PR template.
-- Title the PR with one of the following semantic prefixes (inspired by [Karma](http://karma-runner.github.io/0.10/dev/git-commit-msg.html])):
+- Title the PR with one of the following semantic prefixes (inspired by [Karma](http://karma-runner.github.io/0.10/dev/git-commit-msg.html)):
 
   - `feat` (new feature)
   - `fix` (bug fix)
@@ -663,8 +663,8 @@ tox -e pylint
 
 In terms of best practices please advoid blanket disablement of Pylint messages globally (via `.pylintrc`) or top-level within the file header, albeit there being a few exceptions. Disablement should occur inline as it prevents masking issues and provides context as to why said message is disabled.
 
-Additionally the Python code is auto-formatted using [Black](https://github.com/python/black) which
-is configured as a pre-commit hook. There are also numerous [editor integrations](https://black.readthedocs.io/en/stable/editor_integration.html)
+Additionally, the Python code is auto-formatted using [Black](https://github.com/python/black) which
+is configured as a pre-commit hook. There are also numerous [editor integrations](https://black.readthedocs.io/en/stable/integrations/editors.html)
 
 ### TypeScript
 
@@ -804,7 +804,6 @@ We use [Cypress](https://www.cypress.io/) for integration tests. Tests can be ru
 ```bash
 export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
 export SUPERSET_TESTENV=true
-export ENABLE_REACT_CRUD_VIEWS=true
 export CYPRESS_BASE_URL="http://localhost:8081"
 superset db upgrade
 superset load_test_users

Makefile

@@ -15,8 +15,8 @@
 # limitations under the License.
 #
 
-# Python version installed; we need 3.7-3.9
-PYTHON=`command -v python3.9 || command -v python3.8 || command -v python3.7`
+# Python version installed; we need 3.8-3.9
+PYTHON=`command -v python3.9 || command -v python3.8`
 
 .PHONY: install superset venv pre-commit
 
@@ -70,7 +70,7 @@ update-js:
 
 venv:
 	# Create a virtual environment and activate it (recommended)
-	if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.7, 3.8 or 3.9 installed"; exit 1; fi
+	if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.8 or 3.9 installed"; exit 1; fi
 	test -d venv || ${PYTHON} -m venv venv # setup a python3 virtualenv
 	. venv/bin/activate
 

RELEASING/README.md

@@ -287,6 +287,8 @@ cd ~/src/superset/
 git branch
 # Create the release tag
 git tag -f ${SUPERSET_VERSION}
+# push the tag to the remote
+git push upstream ${SUPERSET_VERSION}
 ```
 
 ### Update CHANGELOG and UPDATING on superset

RELEASING/changelog.py

@@ -381,12 +381,12 @@ def change_log(
         with open(csv, "w") as csv_file:
             log_items = list(logs)
             field_names = log_items[0].keys()
-            writer = lib_csv.DictWriter(  # type: ignore
+            writer = lib_csv.DictWriter(
                 csv_file,
                 delimiter=",",
                 quotechar='"',
                 quoting=lib_csv.QUOTE_ALL,
-                fieldnames=field_names,  # type: ignore
+                fieldnames=field_names,
             )
             writer.writeheader()
             for log in logs:

RESOURCES/FEATURE_FLAGS.md

@@ -41,8 +41,8 @@ These features are **finished** but currently being tested. They are usable, but
 - DYNAMIC_PLUGINS: [(docs)](https://superset.apache.org/docs/installation/running-on-kubernetes)
 - DASHBOARD_NATIVE_FILTERS
 - GLOBAL_ASYNC_QUERIES [(docs)](https://github.com/apache/superset/blob/master/CONTRIBUTING.md#async-chart-queries)
-- OMNIBAR
 - VERSIONED_EXPORT
+- ENABLE_JAVASCRIPT_CONTROLS
 
 ## Stable
 These features flags are **safe for production** and have been tested.
@@ -63,4 +63,3 @@ These features flags currently default to True and **will be removed in a future
 
 - ALLOW_DASHBOARD_DOMAIN_SHARDING
 - DISPLAY_MARKDOWN_HTML
-- ENABLE_REACT_CRUD_VIEWS

UPDATING.md

@@ -24,10 +24,14 @@ assists people when migrating to a new version.
 
 ## Next
 
+- [19046](https://github.com/apache/superset/pull/19046): Enables the drag and drop interface in Explore control panel by default. Flips `ENABLE_EXPLORE_DRAG_AND_DROP` and `ENABLE_DND_WITH_CLICK_UX` feature flags to `True`.
 - [18936](https://github.com/apache/superset/pull/18936): Removes legacy SIP-15 interm logic/flags—specifically the `SIP_15_ENABLED`, `SIP_15_GRACE_PERIOD_END`, `SIP_15_DEFAULT_TIME_RANGE_ENDPOINTS`, and `SIP_15_TOAST_MESSAGE` flags. Time range endpoints are no longer configurable and strictly adhere to the `[start, end)` paradigm, i.e., inclusive of the start and exclusive of the end. Additionally this change removes the now obsolete `time_range_endpoints` from the form-data and resulting in the cache being busted.
 
 ### Breaking Changes
 
+- [19231](https://github.com/apache/superset/pull/19231): The `ENABLE_REACT_CRUD_VIEWS` feature flag has been removed (permanently enabled). Any deployments which had set this flag to false will need to verify that the React views support their use case.
+- [17556](https://github.com/apache/superset/pull/17556): Bumps mysqlclient from v1 to v2
+- [19113](https://github.com/apache/superset/pull/19113): The `ENABLE_JAVASCRIPT_CONTROLS` setting has moved from app config to a feature flag. Any deployments who overrode this setting will now need to override the feature flag from here onward.
 - [18976](https://github.com/apache/superset/pull/18976): When running the app in debug mode, the app will default to use `SimpleCache` for `FILTER_STATE_CACHE_CONFIG` and `EXPLORE_FORM_DATA_CACHE_CONFIG`. When running in non-debug mode, a cache backend will need to be defined, otherwise the application will fail to start. For installations using Redis or other caching backends, it is recommended to use the same backend for both cache configs.
 - [17881](https://github.com/apache/superset/pull/17881): Previously simple adhoc filter values on string columns were stripped of enclosing single and double quotes. To fully support literal quotes in filters, both single and double quotes will no longer be removed from filter values.
 - [17984](https://github.com/apache/superset/pull/17984): Default Flask SECRET_KEY has changed for security reasons. You should always override with your own secret. Set `PREVIOUS_SECRET_KEY` (ex: PREVIOUS_SECRET_KEY = "\2\1thisismyscretkey\1\2\\e\\y\\y\\h") with your previous key and use `superset re-encrypt-secrets` to rotate you current secrets
@@ -36,6 +40,9 @@ assists people when migrating to a new version.
 - [18970](https://github.com/apache/superset/pull/18970): Changes feature
 flag for the legacy datasource editor (DISABLE_LEGACY_DATASOURCE_EDITOR) in config.py to True, thus disabling the feature from being shown in the client.
 - [19083](https://github.com/apache/superset/pull/19083): Updates the mutator function in the config file to take a sql argument and a list of kwargs. Any `SQL_QUERY_MUTATOR` config function overrides will need to be updated to match the new set of params. It is advised regardless of the dictionary args that you list in your function arguments, to keep **kwargs as the last argument to allow for any new kwargs to be passed in.
+- [19017](https://github.com/apache/superset/pull/19017): Removes Python 3.7 support.
+- [19142](https://github.com/apache/superset/pull/19142): Changes feature flag for versioned export(VERSIONED_EXPORT) to be true.
+- [19107](https://github.com/apache/superset/pull/19107): Feature flag `SQLLAB_BACKEND_PERSISTENCE` is now on by default, which enables persisting SQL Lab tabs in the backend instead of the browser's `localStorage`.
 
 ### Potential Downtime
 
@@ -48,14 +55,26 @@ flag for the legacy datasource editor (DISABLE_LEGACY_DATASOURCE_EDITOR) in conf
 
 ### Deprecations
 
+- [19078](https://github.com/apache/superset/pull/19078): Creation of old shorturl links has been deprecated in favor of a new permalink feature that solves the long url problem (old shorturls will still work, though!). By default, new permalinks use UUID4 as the key. However, to use serial ids similar to the old shorturls, add the following to your `superset_config.py`: `PERMALINK_KEY_TYPE = "id"`.
 - [18960](https://github.com/apache/superset/pull/18960): Persisting URL params in chart metadata is no longer supported. To set a default value for URL params in Jinja code, use the optional second argument: `url_param("my-param", "my-default-value")`.
 
 ### Other
 
-- [17589](https://github.com/apache/incubator-superset/pull/17589): It is now possible to limit access to users' recent activity data by setting the `ENABLE_BROAD_ACTIVITY_ACCESS` config flag to false, or customizing the `raise_for_user_activity_access` method in the security manager.
+- [17589](https://github.com/apache/superset/pull/17589): It is now possible to limit access to users' recent activity data by setting the `ENABLE_BROAD_ACTIVITY_ACCESS` config flag to false, or customizing the `raise_for_user_activity_access` method in the security manager.
 - [17536](https://github.com/apache/superset/pull/17536): introduced a key-value endpoint to store dashboard filter state. This endpoint is backed by Flask-Caching and the default configuration assumes that the values will be stored in the file system. If you are already using another cache backend like Redis or Memchached, you'll probably want to change this setting in `superset_config.py`. The key is `FILTER_STATE_CACHE_CONFIG` and the available settings can be found in Flask-Caching [docs](https://flask-caching.readthedocs.io/en/latest/).
 - [17882](https://github.com/apache/superset/pull/17882): introduced a key-value endpoint to store Explore form data. This endpoint is backed by Flask-Caching and the default configuration assumes that the values will be stored in the file system. If you are already using another cache backend like Redis or Memchached, you'll probably want to change this setting in `superset_config.py`. The key is `EXPLORE_FORM_DATA_CACHE_CONFIG` and the available settings can be found in Flask-Caching [docs](https://flask-caching.readthedocs.io/en/latest/).
 
+## 1.4.1
+
+### Breaking Changes
+- [17984](https://github.com/apache/superset/pull/17984): Default Flask SECRET_KEY has changed for security reasons. You should always override with your own secret. Set `PREVIOUS_SECRET_KEY` (ex: PREVIOUS_SECRET_KEY = "\2\1thisismyscretkey\1\2\\e\\y\\y\\h") with your previous key and use `superset re-encrypt-secrets` to rotate you current secrets
+
+### Potential Downtime
+
+### Deprecations
+
+### Other
+
 ## 1.4.0
 
 ### Breaking Changes

docker/docker-bootstrap.sh

@@ -23,7 +23,6 @@ REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
 if [ "$CYPRESS_CONFIG" == "true" ]; then
     export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
     export SUPERSET_TESTENV=true
-    export ENABLE_REACT_CRUD_VIEWS=true
     export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
 fi
 #

docker/docker-init.sh

@@ -43,7 +43,6 @@ if [ "$CYPRESS_CONFIG" == "true" ]; then
     ADMIN_PASSWORD="general"
     export SUPERSET_CONFIG=tests.superset_test_config
     export SUPERSET_TESTENV=true
-    export ENABLE_REACT_CRUD_VIEWS=true
     export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
 fi
 # Initialize the database

docs/docs/contributing/hooks-and-linting.mdx

@@ -41,8 +41,8 @@ tox -e pylint
 
 In terms of best practices please advoid blanket disablement of Pylint messages globally (via `.pylintrc`) or top-level within the file header, albeit there being a few exceptions. Disablement should occur inline as it prevents masking issues and provides context as to why said message is disabled.
 
-Additionally the Python code is auto-formatted using [Black](https://github.com/python/black) which
-is configured as a pre-commit hook. There are also numerous [editor integrations](https://black.readthedocs.io/en/stable/editor_integration.html)
+Additionally, the Python code is auto-formatted using [Black](https://github.com/python/black) which
+is configured as a pre-commit hook. There are also numerous [editor integrations](https://black.readthedocs.io/en/stable/integrations/editors.html)
 
 ### TypeScript
 

docs/docs/contributing/local-backend.mdx

@@ -12,7 +12,7 @@ version: 1
 Make sure your machine meets the [OS dependencies](https://superset.apache.org/docs/installation/installing-superset-from-scratch#os-dependencies) before following these steps.
 You also need to install MySQL or [MariaDB](https://mariadb.com/downloads).
 
-Ensure that you are using Python version 3.7 or 3.8, then proceed with:
+Ensure that you are using Python version 3.8 or 3.9, then proceed with:
 
 ````bash
 # Create a virtual environment and activate it (recommended)

docs/docs/contributing/testing-locally.mdx

@@ -76,7 +76,6 @@ We use [Cypress](https://www.cypress.io/) for integration tests. Tests can be ru
 ```bash
 export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
 export SUPERSET_TESTENV=true
-export ENABLE_REACT_CRUD_VIEWS=true
 export CYPRESS_BASE_URL="http://localhost:8081"
 superset db upgrade
 superset load_test_users

docs/docs/databases/mysql.mdx

@@ -7,7 +7,7 @@ version: 1
 
 ## MySQL
 
-The recommended connector library for MySQL is `[mysqlclient](https://pypi.org/project/mysqlclient/)`.
+The recommended connector library for MySQL is [mysqlclient](https://pypi.org/project/mysqlclient/).
 
 Here's the connection string:
 

docs/docs/installation/alerts-reports.mdx

@@ -33,6 +33,9 @@ Alerts and reports are disabled by default. To turn them on, you need to do some
 - You must install a headless browser, for taking screenshots of the charts and dashboards. Only Firefox and Chrome are currently supported.
   > If you choose Chrome, you must also change the value of `WEBDRIVER_TYPE` to `"chrome"` in your `superset_config.py`.
 
+Note : All the components required (headless browser, redis, postgres db, celery worker and celery beat) are present in the docker image if you are following [Installing Superset Locally](https://superset.apache.org/docs/installation/installing-superset-using-docker-compose/).
+All you need to do is add the required config (See `Detailed Config`). Set `ALERT_REPORTS_NOTIFICATION_DRY_RUN` to `False` in [superset config](https://github.com/apache/superset/blob/master/docker/pythonpath_dev/superset_config.py) to disable dry-run mode and start receiving email/slack notifications.
+
 #### Slack integration
 
 To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace.