Merge branch 'master' of https://github.com/apache/superset into grid…

…-improv

.codecov.yml

@@ -3,6 +3,7 @@ codecov:
         after_n_builds: 4
 ignore:
   - "superset/migrations/versions/*.py"
+  - "superset-frontend/packages/superset-ui-demo/**/*"
   - "**/*.stories.tsx"
   - "**/*.stories.jsx"
 coverage:
@@ -14,6 +15,17 @@ coverage:
         # project coverage decrease:
         target: auto
         threshold: 0%
+      core-packages-ts:
+        target: 95%
+        paths:
+          - 'superset-frontend/packages'
+          - '!superset-frontend/packages/**/*.jsx'
+          - '!superset-frontend/packages/**/*.tsx'
+      core-packages-tsx:
+        target: 50%
+        paths:
+          - 'superset-frontend/packages/**/*.jsx'
+          - 'superset-frontend/packages/**/*.tsx'
     patch:
       default:
         informational: true

.github/workflows/superset-frontend.yml

@@ -43,11 +43,24 @@ jobs:
         run: |
           npm run lint
           npm run prettier-check
+      - name: Build plugins packages
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend
+        run: npm run plugins:build
+      - name: Build plugins Storybook
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend
+        run: npm run plugins:build-storybook
       - name: unit tests
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend
         run: |
           npm run test -- --coverage
+      # todo: remove this step when fix generator as a project in root jest.config.js
+      - name: generator-superset unit tests
+        if: steps.check.outcome == 'failure'
+        working-directory: ./superset-frontend/packages/generator-superset
+        run: npx jest
       - name: Upload code coverage
         if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend

.gitignore

@@ -60,14 +60,29 @@ tmp
 rat-results.txt
 superset/app/
 
-# Node.js, webpack artifacts
+# Node.js, webpack artifacts, storybook
 *.entry.js
 *.js.map
 node_modules
 npm-debug.log*
 superset/static/assets
 superset/static/version_info.json
+superset-frontend/**/esm/*
+superset-frontend/**/lib/*
+superset-frontend/**/storybook-static/*
 yarn-error.log
+*.map
+*.min.js
+test-changelog.md
+*.tsbuildinfo
+
+# Ignore package-lock in packages
+plugins/*/package-lock.json
+packages/*/package-lock.json
+
+# For country map geojson conversion script
+.ipynb_checkpoints/
+scripts/*.zip
 
 # IntelliJ
 *.iml

Dockerfile

@@ -57,13 +57,11 @@ ENV BUILD_CMD=${NPM_BUILD_CMD}
 RUN mkdir -p /app/superset-frontend
 RUN mkdir -p /app/superset/assets
 COPY ./docker/frontend-mem-nag.sh /
-COPY ./superset-frontend/package* /app/superset-frontend/
+COPY ./superset-frontend /app/superset-frontend
 RUN /frontend-mem-nag.sh \
         && cd /app/superset-frontend \
         && npm ci
 
-# Next, copy in the rest and let webpack do its thing
-COPY ./superset-frontend /app/superset-frontend
 # This seems to be the most expensive step
 RUN cd /app/superset-frontend \
         && npm run ${BUILD_CMD} \

UPDATING.md

@@ -26,6 +26,7 @@ assists people when migrating to a new version.
 
 ### Breaking Changes
 
+- [17556](https://github.com/apache/superset/pull/17556): Bumps mysqlclient from v1 to v2
 - [15254](https://github.com/apache/superset/pull/15254): Previously `QUERY_COST_FORMATTERS_BY_ENGINE`, `SQL_VALIDATORS_BY_ENGINE` and `SCHEDULED_QUERIES` were expected to be defined in the feature flag dictionary in the `config.py` file. These should now be defined as a top-level config, with the feature flag dictionary being reserved for boolean only values.
 - [17290](https://github.com/apache/superset/pull/17290): Bumps pandas to `1.3.4` and pyarrow to `5.0.0`
 - [16660](https://github.com/apache/incubator-superset/pull/16660): The `columns` Jinja parameter has been renamed `table_columns` to make the `columns` query object parameter available in the Jinja context.

helm/superset/Chart.yaml

@@ -22,7 +22,7 @@ maintainers:
   - name: craig-rueda
     email: craig@craigrueda.com
     url: https://github.com/craig-rueda
-version: 0.3.12
+version: 0.4.0
 dependencies:
 - name: postgresql
   version: 10.2.0

helm/superset/templates/init-job.yaml

@@ -26,6 +26,10 @@ spec:
   template:
     metadata:
       name: {{ template "superset.name" . }}-init-db
+      {{- if .Values.init.podAnnotations }}
+      annotations:
+        {{ toYaml .Values.init.podAnnotations | nindent 8 }}
+      {{- end }}
     spec:
       securityContext:
         runAsUser: {{ .Values.runAsUser }}

helm/superset/values.yaml

@@ -335,6 +335,8 @@ init:
       echo "Importing database connections.... "
       superset import_datasources -p {{ .Values.extraConfigMountPath }}/import_datasources.yaml
     fi
+  ## Annotations to be added to init job pods
+  podAnnotations: {}
 ##
 ## Configuration values for the postgresql dependency.
 ## ref: https://github.com/kubernetes/charts/blob/master/stable/postgresql/README.md

requirements/development.in

@@ -17,7 +17,7 @@
 # under the License.
 -r base.in
 flask-cors>=2.0.0
-mysqlclient==2.0.3
+mysqlclient>=2.1.0
 pillow>=8.3.2,<9
 pydruid>=0.6.1,<0.7
 pyhive[hive]>=0.6.1

requirements/development.txt

@@ -1,4 +1,4 @@
-# SHA1:2d3a7d56d8fdec412b556af9b6dd21a3e6711eb0
+# SHA1:c835350e502151c63817980505ce50b9e5b78a54
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -36,7 +36,7 @@ jsonlines==2.0.0
     # via tabulator
 linear-tsv==1.1.0
     # via tabulator
-mysqlclient==2.0.3
+mysqlclient==2.1.0
     # via -r requirements/development.in
 openpyxl==3.0.7
     # via tabulator

scripts/babel_extract.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+LICENSE_TMP=$(mktemp)
+cat <<'EOF'> "$LICENSE_TMP"
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+EOF
+
+cd $ROOT_DIR
+pybabel extract \
+  -F superset/translations/babel.cfg \
+  -o superset/translations/messages.pot \
+  --sort-output \
+  --copyright-holder=Superset \
+  --project=Superset \
+  -k _ -k __ -k t -k tn -k tct .
+cat $LICENSE_TMP $ROOT_DIR/superset/translations/messages.pot > messages.pot.tmp \
+  && mv messages.pot.tmp $ROOT_DIR/superset/translations/messages.pot \
+  && cd $CURRENT_DIR

setup.py

@@ -141,7 +141,7 @@ def get_git_sha() -> str:
         "impala": ["impyla>0.16.2, <0.17"],
         "kylin": ["kylinpy>=2.8.1, <2.9"],
         "mmsql": ["pymssql>=2.1.4, <2.2"],
-        "mysql": ["mysqlclient==1.4.2.post1"],
+        "mysql": ["mysqlclient>=2.1.0, <3"],
         "oracle": ["cx-Oracle>8.0.0, <8.1"],
         "pinot": ["pinotdb>=0.3.3, <0.4"],
         "postgres": ["psycopg2-binary==2.8.5"],

superset-frontend/.eslintignore

@@ -28,4 +28,9 @@ src/dashboard/deprecated/*
 src/temp/*
 **/node_modules
 *.d.ts
+coverage/
+esm/
+lib/
+tmp/
+storybook-static/
 temporary_superset_ui/**/*

superset-frontend/.eslintrc.js

@@ -25,6 +25,18 @@ Object.entries(packageConfig.dependencies).forEach(([pkg]) => {
     importCoreModules.push(pkg);
   }
 });
+
+// ignore files when running ForkTsCheckerWebpackPlugin
+let ignorePatterns = [];
+if (process.env.NODE_ENV === 'production') {
+  ignorePatterns = [
+    '*.test.{js,ts,jsx,tsx}',
+    'plugins/**/test/**/*',
+    'packages/**/test/**/*',
+    'packages/generator-superset/**/*',
+  ];
+}
+
 module.exports = {
   extends: [
     'airbnb',
@@ -307,4 +319,5 @@ module.exports = {
     'react/static-property-placement': 0, // disabled temporarily
     'prettier/prettier': 'error',
   },
+  ignorePatterns,
 };

superset-frontend/babel.config.js

@@ -38,6 +38,14 @@ module.exports = {
       '@babel/preset-react',
       { development: process.env.BABEL_ENV === 'development' },
     ],
+    '@babel/preset-typescript',
+    [
+      '@emotion/babel-preset-css-prop',
+      {
+        autoLabel: 'dev-only',
+        labelFormat: '[local]',
+      },
+    ],
   ],
   plugins: [
     'lodash',
@@ -47,6 +55,8 @@ module.exports = {
     ['@babel/plugin-proposal-private-methods', { loose: true }],
     ['@babel/plugin-proposal-nullish-coalescing-operator', { loose: true }],
     ['@babel/plugin-transform-runtime', { corejs: 3 }],
+    // only used in packages/superset-ui-core/src/chart/components/reactify.tsx
+    ['babel-plugin-typescript-to-proptypes', { loose: true }],
     'react-hot-loader/babel',
   ],
   env: {
@@ -70,7 +80,14 @@ module.exports = {
     },
     // build instrumented code for testing code coverage with Cypress
     instrumented: {
-      plugins: ['istanbul'],
+      plugins: [
+        [
+          'istanbul',
+          {
+            exclude: ['plugins/**/*', 'packages/**/*'],
+          },
+        ],
+      ],
     },
     production: {
       plugins: [

superset-frontend/jest.config.js

@@ -16,33 +16,64 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 module.exports = {
-  testRegex: '(\\/spec|\\/src)\\/.*(_spec|\\.test)\\.(j|t)sx?$',
+  testRegex: '\\/(spec|src|plugins|packages)\\/.*(_spec|\\.test)\\.[jt]sx?$',
+  testPathIgnorePatterns: [
+    'packages\\/generator-superset\\/.*',
+    '/node_modules/',
+  ],
   moduleNameMapper: {
     '\\.(css|less|geojson)$': '<rootDir>/spec/__mocks__/mockExportObject.js',
     '\\.(gif|ttf|eot|png|jpg)$': '<rootDir>/spec/__mocks__/mockExportString.js',
     '\\.svg$': '<rootDir>/spec/__mocks__/svgrMock.tsx',
     '^src/(.*)$': '<rootDir>/src/$1',
     '^spec/(.*)$': '<rootDir>/spec/$1',
+    // mapping to souce code instead of lib or esm module
+    '@superset-ui/(((?!(legacy-preset-chart-deckgl|core/src)).)*)$':
+      '<rootDir>/node_modules/@superset-ui/$1/src',
+    '@superset-ui/core/src/(.*)$':
+      '<rootDir>/node_modules/@superset-ui/core/src/$1',
   },
   testEnvironment: 'jsdom',
   modulePathIgnorePatterns: ['<rootDir>/temporary_superset_ui'],
   setupFilesAfterEnv: ['<rootDir>/spec/helpers/setup.ts'],
   testURL: 'http://localhost',
-  collectCoverageFrom: ['src/**/*.{js,jsx,ts,tsx}', '!**/*.stories.*'],
+  collectCoverageFrom: [
+    'src/**/*.{js,jsx,ts,tsx}',
+    '{packages,plugins}/**/src/**/*.{js,jsx,ts,tsx}',
+    '!**/*.stories.*',
+    '!packages/superset-ui-demo/**/*',
+  ],
   coverageDirectory: '<rootDir>/coverage/',
+  coveragePathIgnorePatterns: [
+    'coverage/',
+    'node_modules/',
+    'public/',
+    'tmp/',
+    'dist/',
+  ],
+  coverageReporters: ['lcov', 'json-summary', 'html'],
   transform: {
     '^.+\\.jsx?$': 'babel-jest',
+    // ts-jest can't load plugin 'babel-plugin-typescript-to-proptypes'
+    'reactify\\.tsx$': 'babel-jest',
     '^.+\\.tsx?$': 'ts-jest',
   },
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
   snapshotSerializers: ['@emotion/jest/enzyme-serializer'],
   globals: {
     'ts-jest': {
       babelConfig: true,
+      // todo: duo to packages/**/test and plugins/**/test lack of type checking
+      // turning off checking in Jest.
+      // https://kulshekhar.github.io/ts-jest/docs/getting-started/options/isolatedModules
+      isolatedModules: true,
       diagnostics: {
         warnOnly: true,
       },
     },
+    __DEV__: true,
+    caches: true,
   },
 };

superset-frontend/lerna.json

@@ -1,7 +1,7 @@
 {
   "lerna": "3.2.1",
   "npmClient": "npm",
-  "packages": ["packages/*", "plugins/*", "temporary_superset_ui/*"],
+  "packages": ["packages/*", "plugins/*"],
   "useWorkspaces": true,
   "version": "0.0.0",
   "ignoreChanges": [