Merge pull request #186 from CybercentreCanada/feature/update-cccs-ma…

…in-to-2.0-cccs-1.5-changes

Feature/update cccs main to 2.0 cccs 1.5 changes

.github/CODEOWNERS

@@ -1,14 +1,19 @@
 # Notify all committers of DB migration changes, per SIP-59
+
 # https://github.com/apache/superset/issues/13351
+
 /superset/migrations/ @apache/superset-committers
 
 # Notify Preset team when ephemeral env settings are changed
+
 .github/workflows/ecs-task-definition.json @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 .github/workflows/docker-ephemeral-env.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 .github/workflows/ephemeral*.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 
 # Notify some committers of changes in the Select component
-/superset-frontend/src/components/Select/ @michael-s-molina @geido
+
+/superset-frontend/src/components/Select/ @michael-s-molina @geido @ktmud
 
 # Notify Helm Chart maintainers about changes in it
+
 /helm/superset/ @craig-rueda @dpgaspar @villebro

.github/ISSUE_TEMPLATE/feature_request.md

@@ -5,14 +5,10 @@ labels: "#enhancement"
 
 ---
 
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+Github Discussions is our new home for discussing features and improvements!
 
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+https://github.com/apache/superset/discussions/categories/ideas
 
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+We'd like to keep Github Issues focuses on bugs and SIP's (Superset Improvement Proposals)!
 
-**Additional context**
-Add any other context or screenshots about the feature request here.
+Please note that feature requests opened as Github Issues will be moved to Discussions.

.github/workflows/bashlib.sh

@@ -38,10 +38,10 @@ default-setup-command() {
 }
 
 apt-get-install() {
-    say "::group::apt-get install dependencies"
-    sudo apt-get update && sudo apt-get install --yes \
-      libsasl2-dev
-    say "::endgroup::"
+  say "::group::apt-get install dependencies"
+  sudo apt-get update && sudo apt-get install --yes \
+    libsasl2-dev
+  say "::endgroup::"
 }
 
 pip-upgrade() {
@@ -161,7 +161,7 @@ cypress-run() {
   if [[ -z $CYPRESS_KEY ]]; then
     $cypress --spec "cypress/integration/$page" --browser "$browser"
   else
-    export CYPRESS_RECORD_KEY=`echo $CYPRESS_KEY | base64 --decode`
+    export CYPRESS_RECORD_KEY=$(echo $CYPRESS_KEY | base64 --decode)
     # additional flags for Cypress dashboard recording
     $cypress --spec "cypress/integration/$page" --browser "$browser" \
       --record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \
@@ -183,22 +183,22 @@ cypress-run-all() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "*/**/*"
+  cypress-run "*/**/!(*.applitools.test.ts)"
 
   # After job is done, print out Flask log for debugging
   say "::group::Flask log for default run"
   cat "$flasklog"
   say "::endgroup::"
 
-  # Rerun SQL Lab tests with backend persist enabled
-  export SUPERSET_CONFIG=tests.integration_tests.superset_test_config_sqllab_backend_persist
+  # Rerun SQL Lab tests with backend persist disabled
+  export SUPERSET_CONFIG=tests.integration_tests.superset_test_config_sqllab_backend_persist_off
 
   # Restart Flask with new configs
   kill $flaskProcessId
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "sqllab/*" "Backend persist"
+  cypress-run "sqllab/!(*.applitools.test.ts)" "Backend persist"
 
   # Upload code coverage separately so each page can have separate flags
   # -c will clean existing coverage reports, -F means add flags
@@ -212,3 +212,29 @@ cypress-run-all() {
   # make sure the program exits
   kill $flaskProcessId
 }
+
+eyes-storybook-dependencies() {
+  say "::group::install eyes-storyook dependencies"
+  sudo apt-get update -y && sudo apt-get -y install gconf-service ca-certificates libxshmfence-dev fonts-liberation libappindicator3-1 libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf-2-4 libglib2.0-0 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release xdg-utils libappindicator1
+  say "::endgroup::"
+}
+
+cypress-run-applitools() {
+  local flasklog="${HOME}/flask.log"
+  local port=8081
+  export CYPRESS_BASE_URL="http://localhost:${port}"
+
+  nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
+  local flaskProcessId=$!
+
+  cypress-run "*/**/*.applitools.test.ts"
+
+  codecov -c -F "cypress" || true
+
+  say "::group::Flask log for default run"
+  cat "$flasklog"
+  say "::endgroup::"
+
+  # make sure the program exits
+  kill $flaskProcessId
+}

.github/workflows/check_db_migration_confict.yml

@@ -50,10 +50,11 @@ jobs:
                   repo: context.repo.repo,
                   issue_number: pull.number,
                   body:
-                    `⚠️ @${pull.user.login} Your base branch \`${currentBranch}\` has just ` +
+                    `# 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️` +
+                    `❗ @${pull.user.login} Your base branch \`${currentBranch}\` has ` +
                     'also updated `superset/migrations`.\n' +
                     '\n' +
-                    '❗ **Please consider rebasing your branch to avoid db migration conflicts.**',
+                    '**Please consider rebasing your branch and [resolving potential db migration conflicts](https://github.com/apache/superset/blob/master/CONTRIBUTING.md#merging-db-migrations).**',
                 });
               }
             }

.github/workflows/superset-applitool-cypress.yml

@@ -0,0 +1,88 @@
+name: Applitools Cypress
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  cypress-applitools:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: ["chrome"]
+        node: [16]
+    env:
+      FLASK_ENV: development
+      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
+      SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+      PYTHONPATH: ${{ github.workspace }}
+      REDIS_PORT: 16379
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      APPLITOOLS_APP_NAME: Superset
+      APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
+      APPLITOOLS_BATCH_ID: ${{ github.sha }}
+      APPLITOOLS_BATCH_NAME: Superset Cypress
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        ports:
+          - 15432:5432
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          submodules: recursive
+          ref: master
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: OS dependencies
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: apt-get-install
+      - name: Install python dependencies
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: |
+            pip-upgrade
+            pip install -r requirements/testing.txt
+      - name: Setup postgres
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: setup-postgres
+      - name: Import test data
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: testdata
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node }}
+      - name: Install npm dependencies
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: npm-install
+      - name: Build javascript packages
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: build-instrumented-assets
+      - name: Install cypress
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: cypress-install
+      - name: Run Cypress
+        uses: ./.github/actions/cached-dependencies
+        env:
+          CYPRESS_BROWSER: ${{ matrix.browser }}
+        with:
+          run: cypress-run-applitools

.github/workflows/superset-applitools-storybook.yml

@@ -0,0 +1,40 @@
+name: Applitools Storybook
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+env:
+  APPLITOOLS_APP_NAME: Superset
+  APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
+  APPLITOOLS_BATCH_ID: ${{ github.sha }}
+  APPLITOOLS_BATCH_NAME: Superset Storybook
+
+jobs:
+  cron:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        node: [16]
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          submodules: recursive
+          ref: master
+      - name: Set up Node.js
+        uses: actions/setup-node@v3.1.1
+        with:
+          node-version: ${{ matrix.node }}
+      - name: Install eyes-storybook dependencies
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: eyes-storybook-dependencies
+      - name: Install NPM dependencies
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: npm-install
+      - name: Run Applitools Eyes-Storybook
+        working-directory: ./superset-frontend
+        run: npx eyes-storybook -u https://superset-storybook.netlify.app/

.github/workflows/superset-cli.yml

@@ -0,0 +1,76 @@
+name: Superset CLI tests
+
+on:
+  push:
+    branches-ignore:
+      - "dependabot/npm_and_yarn/**"
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  test-load-examples:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.9]
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
+      REDIS_PORT: 16379
+      SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        ports:
+          # Use custom ports for services to avoid accidentally connecting to
+          # GitHub action runner's default installations
+          - 15432:5432
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
+      - name: Setup Python
+        if: steps.check.outcome == 'failure'
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'requirements/testing.txt'
+      - name: Install dependencies
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: |
+            apt-get-install
+            pip-upgrade
+            pip install wheel
+            pip install -r requirements/testing.txt
+            setup-postgres
+      - name: superset init
+        if: steps.check.outcome == 'failure'
+        run: |
+          pip install -e .
+          superset db upgrade
+          superset load_test_users
+      - name: superset load_examples
+        if: steps.check.outcome == 'failure'
+        run: |
+          # load examples without test data
+          superset load_examples --load-big-data

.github/workflows/superset-e2e.yml

@@ -24,15 +24,14 @@ jobs:
         browser: ["chrome"]
     env:
       FLASK_ENV: development
-      ENABLE_REACT_CRUD_VIEWS: true
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
       SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
       PYTHONPATH: ${{ github.workspace }}
       REDIS_PORT: 16379
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     services:
       postgres:
-        image: postgres:10-alpine
+        image: postgres:14-alpine
         env:
           POSTGRES_USER: superset
           POSTGRES_PASSWORD: superset
@@ -93,7 +92,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v2
         with:
-          node-version:  '16'
+          node-version: "16"
       - name: Install npm dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

.github/workflows/superset-python-integrationtest.yml

@@ -88,7 +88,7 @@ jobs:
       SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
     services:
       postgres:
-        image: postgres:10-alpine
+        image: postgres:14-alpine
         env:
           POSTGRES_USER: superset
           POSTGRES_PASSWORD: superset